From 2a061ba865d2509decc762f11fbdbcf2871d04f9 Mon Sep 17 00:00:00 2001 From: Tom Parker-Shemilt Date: Wed, 15 Jul 2026 18:14:34 +0100 Subject: [PATCH 01/84] Adds weekly and "everything" docs regen (#2551) --- .github/workflows/config-reference.yaml | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/.github/workflows/config-reference.yaml b/.github/workflows/config-reference.yaml index bc8516e62..965209f43 100644 --- a/.github/workflows/config-reference.yaml +++ b/.github/workflows/config-reference.yaml @@ -26,9 +26,13 @@ on: workflow_dispatch: inputs: tag: - description: 'Existing release tag to regenerate the config reference for (e.g. v1.6.1).' - required: true + description: 'Existing release tag to regenerate the config reference for (e.g. v1.6.1) or blank for all' + required: false # Blank to allow full regen type: string + # Weekly docs regeneration to catch any cases where we've made updates but haven't changed anything + schedule: + # Sunday 4:13am randomly picked + - cron: 13 4 * * SUN permissions: read-all @@ -52,10 +56,12 @@ jobs: set -euo pipefail if [[ "${{ github.event_name }}" == "workflow_dispatch" ]]; then TAG="${{ inputs.tag }}" + elif [[ "${{ github.event_name }}" == "schedule" ]]; then + TAG= else TAG="${{ github.ref_name }}" fi - if [[ ! "${TAG}" =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]]; then + if [[ "${TAG}" != "" && ( ! "${TAG}" =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ) ]]; then echo "::error::'${TAG}' is not a release tag (vMAJOR.MINOR.PATCH); refusing to regenerate." exit 1 fi From 27fb8adde6554de68d452ef87535822ad4330680 Mon Sep 17 00:00:00 2001 From: Tom Parker-Shemilt Date: Thu, 16 Jul 2026 11:25:44 +0100 Subject: [PATCH 02/84] Enforce Vale settings for config as well as markdown (#2552) --- .../vocabularies/TraceMachina/accept.txt | 24 ++++++++-- .vale.ini | 45 ++++++++++++++++++ .../scheduler-multi-worker.json5 | 4 +- .../docker-compose/scheduler.json5 | 4 +- .../docker-compose/worker-shared-cas.json5 | 4 +- .../docker-compose/worker.json5 | 4 +- kubernetes/components/worker/worker.json5 | 4 +- nativelink-config/src/cas_server.rs | 20 ++++---- nativelink-config/src/schedulers.rs | 18 +++---- nativelink-config/src/stores.rs | 47 ++++++++++--------- .../src/worker_capability_index.rs | 2 +- .../redis_store_awaited_action_db_test.rs | 6 +-- nativelink-service/tests/cas_server_test.rs | 8 ++-- nativelink-store/src/shard_store.rs | 2 +- nativelink-store/tests/memory_store_test.rs | 4 +- nativelink-util/src/telemetry.rs | 7 ++- 16 files changed, 132 insertions(+), 71 deletions(-) diff --git a/.github/styles/config/vocabularies/TraceMachina/accept.txt b/.github/styles/config/vocabularies/TraceMachina/accept.txt index 65d05bcbe..4725cfdb8 100644 --- a/.github/styles/config/vocabularies/TraceMachina/accept.txt +++ b/.github/styles/config/vocabularies/TraceMachina/accept.txt @@ -102,12 +102,11 @@ Config Grafana GitHub Deno -Http +HTTP IPs Noop ONTAP -Redis -redis +[Rr]edis shipstorm hermiticity performant @@ -172,7 +171,7 @@ impl [Rr]epos rules_cc rules_go -Tls +TLS Zlib backplanes deserialize @@ -296,3 +295,20 @@ shellexpand pluggable [Cc]allout hostnames +Serde +camelCase +SDK +varint +prost +unstorable +stdout +args +uncomment +hashmap +Buildbarn +deserialization +Dockerised +Merkle +subtree +hardlink +multiplicatively diff --git a/.vale.ini b/.vale.ini index 07da1def9..376e5cc25 100644 --- a/.vale.ini +++ b/.vale.ini @@ -12,6 +12,51 @@ IgnoredScopes = code, tt, frontmatter [formats] mdx = md +rs = md + +# FIXME(palfrey): really this should be all our rust code, but that's a lot of changes +# Focusing first on the config bits that get generated into our reference code +# When we do that, we can de-dupe this section and the markdown-specific one +[nativelink-config/src/*.rs] +BasedOnStyles = alex, Vale, Microsoft, write-good + +# Ignore code blocks in Starlight's TabItems. +BlockIgnores = (?s)(.*?```.*?```.*?) + +# Skip URL targets in Markdown links — Vale.Terms shouldn't lint URLs, +# but does by default. The `(?<=\])` lookbehind narrows the pattern so +# it only matches the URL portion of `[link text](url)`. +TokenIgnores = ['"]\.\/[\w-]+\.mdx['"], (?<=\])\([^)]+\) + +# Too harsh. The `write-good.Passive` check already covers many cases. +write-good.E-Prime = NO + +# Redundant. Covered by `write-good.Passive`. +Microsoft.Passive = NO + +# We use typographic em-dashes with spaces for readability. The Microsoft +# style guide wants tight em-dashes; we don't. +Microsoft.Dashes = NO + +# Pedantic style rules we don't want to enforce in technical docs: +Microsoft.Avoid = NO # blanket bans on words like "backend" +Microsoft.HeadingColons = NO # nitpicks heading punctuation +Microsoft.Contractions = NO # "what's" vs "what is" is a tone choice +Microsoft.Foreign = NO # allows "e.g." and "i.e." +Microsoft.Quotes = NO # punctuation-inside-quotes is style +Microsoft.We = NO # we use "we" in conversational tech docs +Microsoft.Negative = NO # en-dash isn't a good idea for negatives + +# Vale's built-in Terms rule does case-sensitive substitution. In tech +# docs, the same word can be a lowercase command (`bazel build`), a +# capitalised proper noun (Bazel), or a path fragment in a URL. Trying +# to enforce one form across all three is noisy and produces false +# positives in code blocks and URLs. +Vale.Terms = NO + +# Microsoft.Auto wants "autodetects" instead of "auto-detects". Both are +# fine in technical prose. +Microsoft.Auto = NO [*.{md,mdx}] BasedOnStyles = alex, Vale, Microsoft, write-good diff --git a/deployment-examples/docker-compose/scheduler-multi-worker.json5 b/deployment-examples/docker-compose/scheduler-multi-worker.json5 index 18a28333f..380f73b6f 100644 --- a/deployment-examples/docker-compose/scheduler-multi-worker.json5 +++ b/deployment-examples/docker-compose/scheduler-multi-worker.json5 @@ -3,7 +3,7 @@ { name: "GRPC_LOCAL_STORE", - // Note: This file is used to test GRPC store. + // Note: This file is used to test gRPC store. grpc: { instance_name: "", endpoints: [ @@ -17,7 +17,7 @@ { name: "GRPC_LOCAL_AC_STORE", - // Note: This file is used to test GRPC store. + // Note: This file is used to test gRPC store. grpc: { instance_name: "", endpoints: [ diff --git a/deployment-examples/docker-compose/scheduler.json5 b/deployment-examples/docker-compose/scheduler.json5 index 18a28333f..380f73b6f 100644 --- a/deployment-examples/docker-compose/scheduler.json5 +++ b/deployment-examples/docker-compose/scheduler.json5 @@ -3,7 +3,7 @@ { name: "GRPC_LOCAL_STORE", - // Note: This file is used to test GRPC store. + // Note: This file is used to test gRPC store. grpc: { instance_name: "", endpoints: [ @@ -17,7 +17,7 @@ { name: "GRPC_LOCAL_AC_STORE", - // Note: This file is used to test GRPC store. + // Note: This file is used to test gRPC store. grpc: { instance_name: "", endpoints: [ diff --git a/deployment-examples/docker-compose/worker-shared-cas.json5 b/deployment-examples/docker-compose/worker-shared-cas.json5 index 1198cde34..7c8aedc49 100644 --- a/deployment-examples/docker-compose/worker-shared-cas.json5 +++ b/deployment-examples/docker-compose/worker-shared-cas.json5 @@ -3,7 +3,7 @@ { name: "GRPC_LOCAL_STORE", - // Note: This file is used to test GRPC store. + // Note: This file is used to test gRPC store. grpc: { instance_name: "", endpoints: [ @@ -17,7 +17,7 @@ { name: "GRPC_LOCAL_AC_STORE", - // Note: This file is used to test GRPC store. + // Note: This file is used to test gRPC store. grpc: { instance_name: "", endpoints: [ diff --git a/deployment-examples/docker-compose/worker.json5 b/deployment-examples/docker-compose/worker.json5 index 7e55b0f17..ae2afb881 100644 --- a/deployment-examples/docker-compose/worker.json5 +++ b/deployment-examples/docker-compose/worker.json5 @@ -3,7 +3,7 @@ { name: "GRPC_LOCAL_STORE", - // Note: This file is used to test GRPC store. + // Note: This file is used to test gRPC store. grpc: { instance_name: "", endpoints: [ @@ -17,7 +17,7 @@ { name: "GRPC_LOCAL_AC_STORE", - // Note: This file is used to test GRPC store. + // Note: This file is used to test gRPC store. grpc: { instance_name: "", endpoints: [ diff --git a/kubernetes/components/worker/worker.json5 b/kubernetes/components/worker/worker.json5 index d68c57d55..01f729bb3 100644 --- a/kubernetes/components/worker/worker.json5 +++ b/kubernetes/components/worker/worker.json5 @@ -3,7 +3,7 @@ { name: "GRPC_LOCAL_STORE", - // Note: This file is used to test GRPC store. + // Note: This file is used to test gRPC store. grpc: { instance_name: "", endpoints: [ @@ -17,7 +17,7 @@ { name: "GRPC_LOCAL_AC_STORE", - // Note: This file is used to test GRPC store. + // Note: This file is used to test gRPC store. grpc: { instance_name: "", endpoints: [ diff --git a/nativelink-config/src/cas_server.rs b/nativelink-config/src/cas_server.rs index de7213f1b..07a3d233b 100644 --- a/nativelink-config/src/cas_server.rs +++ b/nativelink-config/src/cas_server.rs @@ -352,7 +352,7 @@ pub struct ByteStreamConfig { } // Older bytestream config. All fields are as per the newer docs, but this requires -// the hashed cas_stores v.s. the WithInstanceName approach. This should _not_ be updated +// the hashed `cas_stores` v.s. the WithInstanceName approach. This should _not_ be updated // with newer fields, and eventually dropped #[derive(Deserialize, Serialize, Debug, Clone)] #[serde(deny_unknown_fields)] @@ -488,7 +488,7 @@ pub struct ServicesConfig { /// Capabilities service is required in order to use most of the /// bazel protocol. This service is used to provide the supported - /// features and versions of this bazel GRPC service. + /// features and versions of this bazel gRPC service. #[serde( default, deserialize_with = "super::backcompat::opt_vec_with_instance_name" @@ -878,7 +878,7 @@ pub struct UploadActionResultConfig { pub struct LocalWorkerConfig { /// Name of the worker. This is give a more friendly name to a worker for logging /// and metric publishing. This is also the prefix of the worker id - /// (ie: "{name}{uuidv6}"). + /// (i.e. "{name}{uuidv6}"). /// Default: {Index position in the workers list} #[serde(default, deserialize_with = "convert_string_with_shellexpand")] pub name: String, @@ -994,7 +994,7 @@ pub struct LocalWorkerConfig { pub platform_properties: HashMap, /// An optional mapping of environment names to set for the execution - /// as well as those specified in the action itself. If set, will set each + /// as well as those specified in the action itself. If set, will set each /// key as an environment variable before executing the job with the value /// of the environment variable being the value of the property of the /// action being executed of that name or the fixed value. @@ -1006,9 +1006,9 @@ pub struct LocalWorkerConfig { /// Default: None (directory cache disabled) pub directory_cache: Option, - /// Whether to use namespaces to isolate the execution. This is only available - /// on Linux. It is highly recommended as it avoids a number of issues with - /// zombie processes and also provides additional hermeticity. If explicitly set + /// Whether to use namespaces to isolate the execution. This is only available + /// on Linux. It is highly recommended as it avoids a number of issues with + /// zombie processes and also provides additional hermeticity. If explicitly set /// to true and it is not supported the worker will exit with an error. /// /// Note: this will fail for non-privileged Dockerised workers, as workers in @@ -1018,9 +1018,9 @@ pub struct LocalWorkerConfig { /// Default: False. pub use_namespaces: Option, - /// Whether to use a mount namespace to isolate the worker root. This is only - /// available on Linux and when `use_namespaces` is true. It is highly recommended - /// provides additional hermeticity. If explicitly set to true and it is not + /// Whether to use a mount namespace to isolate the worker root. This is only + /// available on Linux and when `use_namespaces` is true. It is highly recommended + /// provides additional hermeticity. If explicitly set to true and it is not /// supported or `use_namespaces` is not set to true the worker will exit with an /// error. /// Default: False. diff --git a/nativelink-config/src/schedulers.rs b/nativelink-config/src/schedulers.rs index 3d32c9f33..1f74832cd 100644 --- a/nativelink-config/src/schedulers.rs +++ b/nativelink-config/src/schedulers.rs @@ -100,7 +100,7 @@ pub struct SimpleSpec { /// { "cpu_count": "8", "cpu_arch": "arm" } /// ``` /// Will result in the scheduler filtering out any workers that do not have - /// `"cpu_arch" = "arm"` and filter out any workers that have less than 8 cpu + /// `"cpu_arch" = "arm"` and filter out any workers that have less than 8 CPU /// cores available. /// /// The property names here must match the property keys provided by the @@ -108,7 +108,7 @@ pub struct SimpleSpec { /// publish their capabilities to the scheduler when they join the worker /// pool. If the worker fails to notify the scheduler of its (for example) /// `"cpu_arch"`, the scheduler will never send any jobs to it, if all jobs - /// have the `"cpu_arch"` label. There is no special treatment of any platform + /// have the `"cpu_arch"` label. We have no special treatment of any platform /// property labels other and entirely driven by worker configs and this /// config. pub supported_platform_properties: Option>, @@ -188,9 +188,9 @@ pub struct ExperimentalRedisSchedulerBackend { pub redis_store: StoreRefName, } -/// A scheduler that simply forwards requests to an upstream scheduler. This +/// A scheduler that forwards requests to an upstream scheduler. This /// is useful to use when doing some kind of local action cache or CAS away from -/// the main cluster of workers. In general, it's more efficient to point the +/// the main cluster of workers. In general, it's more efficient to point the /// build at the main scheduler directly though. #[derive(Deserialize, Serialize, Debug)] #[serde(deny_unknown_fields)] @@ -203,8 +203,8 @@ pub struct GrpcSpec { #[serde(default)] pub retry: Retry, - /// Limit the number of simultaneous upstream requests to this many. A - /// value of zero is treated as unlimited. If the limit is reached the + /// Limit the number of simultaneous upstream requests to this many. A + /// value of zero is treated as unlimited. If the limit is reached the /// request is queued. /// Default: unlimited #[serde(default, deserialize_with = "convert_numeric_with_shellexpand")] @@ -246,7 +246,7 @@ pub struct PlatformPropertyAddition { pub struct PlatformPropertyReplacement { /// The name of the property to replace. pub name: String, - /// The the value to match against, if unset then any instance matches. + /// The value to match against, if unset then any instance matches. #[serde(default)] pub value: Option, /// The new name of the property. @@ -273,9 +273,9 @@ pub enum PropertyModification { #[cfg_attr(feature = "dev-schema", derive(JsonSchema))] pub struct PropertyModifierSpec { /// A list of modifications to perform to incoming actions for the nested - /// scheduler. These are performed in order and blindly, so removing a + /// scheduler. These are performed in order and blindly, so removing a /// property that doesn't exist is fine and overwriting an existing property - /// is also fine. If adding properties that do not exist in the nested + /// is also fine. If adding properties that do not exist in the nested /// scheduler is not supported and will likely cause unexpected behaviour. pub modifications: Vec, diff --git a/nativelink-config/src/stores.rs b/nativelink-config/src/stores.rs index cfa69715b..d811b01a2 100644 --- a/nativelink-config/src/stores.rs +++ b/nativelink-config/src/stores.rs @@ -438,7 +438,7 @@ pub enum StoreSpec { /// WARNING: If you need data to always exist in the `slow` store /// for something like remote execution, be careful because this /// store will never check to see if the objects exist in the - /// `slow` store if it exists in the `fast` store (ie: it assumes + /// `slow` store if it exists in the `fast` store (i.e. it assumes /// that if an object exists in the `fast` store it will exist in /// the `slow` store). /// @@ -531,7 +531,7 @@ pub enum StoreSpec { /// used if the size field is the real size of the content, in other /// words, don't use on AC (Action Cache) stores. Any store where you can /// safely use `VerifySpec.verify_size = true`, this store should be safe - /// to use (ie: CAS stores). + /// to use (i.e. CAS stores). /// /// **Example JSON Config:** /// ```json @@ -553,7 +553,7 @@ pub enum StoreSpec { /// SizePartitioning(Box), - /// This store will pass-through calls to another GRPC store. This store + /// This store will pass-through calls to another gRPC store. This store /// is not designed to be used as a sub-store of another store, but it /// does satisfy the interface and will likely work. /// @@ -707,7 +707,7 @@ pub struct RefSpec { pub struct FilesystemSpec { /// Path on the system where to store the actual content. This is where /// the bulk of the data will be placed. - /// On service bootup this folder will be scanned and all files will be + /// On service boot this folder will be scanned and all files will be /// added to the cache. In the event one of the files doesn't match the /// criteria, the file will be deleted. #[serde(deserialize_with = "convert_string_with_shellexpand")] @@ -716,7 +716,7 @@ pub struct FilesystemSpec { /// A temporary location of where files that are being uploaded or /// deleted will be placed while the content cannot be guaranteed to be /// accurate. This location must be on the same block device as - /// `content_path` so atomic moves can happen (ie: move without copy). + /// `content_path` so atomic moves can happen (i.e. move without copy). /// All files in this folder will be deleted on every startup. #[serde(deserialize_with = "convert_string_with_shellexpand")] pub temp_path: String, @@ -883,7 +883,7 @@ pub struct FastSlowSpec { /// out to the `slow` store. pub fast: StoreSpec, - /// How to handle the fast store. This can be useful to set to Get for + /// How to handle the fast store. This can be useful to set to Get for /// worker nodes such that results are persisted to the slow store only. #[serde(default)] pub fast_direction: StoreDirection, @@ -892,7 +892,7 @@ pub struct FastSlowSpec { /// get it from this store. pub slow: StoreSpec, - /// How to handle the slow store. This can be useful if creating a diode + /// How to handle the slow store. This can be useful if creating a diode /// and you wish to have an upstream read only store. #[serde(default)] pub slow_direction: StoreDirection, @@ -963,7 +963,7 @@ pub struct DedupSpec { /// Due to implementation detail, we want to prefer to download /// the first chunks of the file so we can stream the content /// out and free up some of our buffers. This configuration - /// will be used to to restrict the number of concurrent chunk + /// will be used to restrict the number of concurrent chunk /// downloads at a time per `get()` request. /// /// This setting will also affect how much memory might be used @@ -1098,7 +1098,7 @@ pub struct EvictionPolicy { pub max_bytes: usize, /// When eviction starts based on hitting `max_bytes`, continue until - /// `max_bytes - evict_bytes` is met to create a low watermark. This stops + /// `max_bytes - evict_bytes` is met to create a low watermark. This stops /// operations from thrashing when the store is close to the limit. /// Default: 0 #[serde(default, deserialize_with = "convert_data_size_with_shellexpand")] @@ -1274,10 +1274,10 @@ pub struct CommonObjectSpec { #[serde(default, deserialize_with = "convert_boolean_with_shellexpand")] pub insecure_allow_http: bool, - /// Disable http/2 connections and only use http/1.1. Default client - /// configuration will have http/1.1 and http/2 enabled for connection - /// schemes. Http/2 should be disabled if environments have poor support - /// or performance related to http/2. Safe to keep default unless + /// Disable HTTP/2 connections and only use HTTP/1.1. Default client + /// configuration will have HTTP/1.1 and HTTP/2 enabled for connection + /// schemes. HTTP/2 should be disabled if environments have poor support + /// or performance related to HTTP/2. Safe to keep default unless /// underlying network environment, S3, or GCS API servers specify otherwise. /// /// Default: false @@ -1327,7 +1327,7 @@ pub struct ClientTlsConfig { #[serde(deny_unknown_fields)] #[cfg_attr(feature = "dev-schema", derive(JsonSchema))] pub struct GrpcEndpoint { - /// The endpoint address (i.e. grpc(s)://example.com:443). + /// The endpoint address (i.e. `grpc(s)://example.com:443`). #[serde(deserialize_with = "convert_string_with_shellexpand")] pub address: String, /// The TLS configuration to use to connect to the endpoint (if grpcs). @@ -1367,7 +1367,7 @@ pub struct GrpcEndpoint { #[serde(deny_unknown_fields)] #[cfg_attr(feature = "dev-schema", derive(JsonSchema))] pub struct GrpcSpec { - /// Instance name for GRPC calls. Proxy calls will have the `instance_name` changed to this. + /// Instance name for gRPC calls. Proxy calls will have the `instance_name` changed to this. #[serde(default, deserialize_with = "convert_string_with_shellexpand")] pub instance_name: String, @@ -1381,19 +1381,20 @@ pub struct GrpcSpec { #[serde(default)] pub retry: Retry, - /// Limit the number of simultaneous upstream requests to this many. A - /// value of zero is treated as unlimited. If the limit is reached the + /// Limit the number of simultaneous upstream requests to this many. A + /// value of zero is treated as unlimited. If the limit is reached the /// request is queued. #[serde(default, deserialize_with = "convert_numeric_with_shellexpand")] pub max_concurrent_requests: usize, /// The number of connections to make to each specified endpoint to balance - /// the load over multiple TCP connections. Default 1. + /// the load over multiple TCP connections. + /// Default: 1. #[serde(default, deserialize_with = "convert_numeric_with_shellexpand")] pub connections_per_endpoint: usize, /// Maximum time (seconds) allowed for a single RPC request (e.g. a - /// ByteStream.Write call) before it is cancelled. + /// `ByteStream.Write` call) before it is cancelled. /// /// A value of 0 (the default) disables the per-RPC timeout. Dead /// connections are still detected by the HTTP/2 and TCP keepalive @@ -1558,7 +1559,7 @@ pub struct RedisSpec { #[serde(default, deserialize_with = "convert_numeric_with_shellexpand")] pub read_chunk_size: usize, - /// The number of connections to keep open to the redis server(s). + /// The number of connections to keep open to the redis servers. /// /// Default: 3 #[serde(default, deserialize_with = "convert_numeric_with_shellexpand")] @@ -1569,7 +1570,7 @@ pub struct RedisSpec { /// large objects to the redis server. A good rule of thumb is to /// think of the data as: /// `AVAIL_MEMORY / (read_chunk_size * max_chunk_uploads_per_update) = THORETICAL_MAX_CONCURRENT_UPLOADS` - /// (note: it is a good idea to divide `AVAIL_MAX_MEMORY` by ~10 to account for other memory usage) + /// (note: it's a good idea to divide `AVAIL_MAX_MEMORY` by ~10 to account for other memory usage) /// /// Default: 10 #[serde(default, deserialize_with = "convert_numeric_with_shellexpand")] @@ -1663,8 +1664,8 @@ pub struct Retry { #[serde(default)] pub jitter: f32, - /// A list of error codes to retry on, if this is not set then the default - /// error codes to retry on are used. These default codes are the most + /// A list of error codes to retry on, if this isn't set then the default + /// error codes to retry on are used. These default codes are the most /// likely to be non-permanent. /// - `Unknown` /// - `Cancelled` diff --git a/nativelink-scheduler/src/worker_capability_index.rs b/nativelink-scheduler/src/worker_capability_index.rs index b0e45b76b..1453a1217 100644 --- a/nativelink-scheduler/src/worker_capability_index.rs +++ b/nativelink-scheduler/src/worker_capability_index.rs @@ -91,7 +91,7 @@ impl WorkerCapabilityIndex { .insert(worker_id.clone()); } PlatformPropertyValue::Minimum(_) | PlatformPropertyValue::Ignore(_) => { - // Minimum properties are tracked via property_presence only. + // Minimum properties are tracked via `property_presence` only. // Their actual values are checked at runtime since they're dynamic. // Ignore properties we just drop diff --git a/nativelink-scheduler/tests/redis_store_awaited_action_db_test.rs b/nativelink-scheduler/tests/redis_store_awaited_action_db_test.rs index 0fd6e4c94..be75aed43 100644 --- a/nativelink-scheduler/tests/redis_store_awaited_action_db_test.rs +++ b/nativelink-scheduler/tests/redis_store_awaited_action_db_test.rs @@ -116,7 +116,7 @@ fn make_awaited_action(operation_id: &str) -> AwaitedAction { ) } -// TODO: This test needs to be rewritten to use workers (like test_multiple_clients_subscribe_to_same_action). +// TODO: This test needs to be rewritten to use workers (like `test_multiple_clients_subscribe_to_same_action`). #[nativelink_test] #[ignore = "needs rewrite to use workers (like test_multiple_clients_subscribe_to_same_action)"] async fn add_action_smoke_test() -> Result<(), Error> { @@ -288,7 +288,7 @@ async fn test_multiple_clients_subscribe_to_same_action() -> Result<(), Error> { .await .unwrap(); - // Second client should be able to get the action by its client_operation_id + // Second client should be able to get the action by its `client_operation_id` let get_subscription = scheduler .filter_operations(OperationFilter { client_operation_id: Some(OperationId::from(CLIENT_OPERATION_ID_2)), @@ -471,7 +471,7 @@ async fn test_orphaned_client_operation_id_returns_none() -> Result<(), Error> { fake_redis_backend.set_subscription_manager(store.subscription_manager().await.unwrap()); // Manually set up the orphaned state in the fake backend: - // 1. Add client_id → operation_id mapping (cid_* key) + // 1. Add `client_id` → `operation_id` mapping (cid_* key) { let mut table = fake_redis_backend.table.lock().unwrap(); let mut client_fields = HashMap::new(); diff --git a/nativelink-service/tests/cas_server_test.rs b/nativelink-service/tests/cas_server_test.rs index f8352ad2b..f61095d92 100644 --- a/nativelink-service/tests/cas_server_test.rs +++ b/nativelink-service/tests/cas_server_test.rs @@ -1597,7 +1597,7 @@ async fn chunking_on_grpc_store_forbids_index_store() -> Result<(), Box Result<(), Box Result<(), Box> { - // avg 1024 (min allowed) with max_chunk_count 2: the 16 KiB test blob + // avg 1024 (min allowed) with `max_chunk_count` 2: the 16 KiB test blob // chunks to more than 2 pieces, so on-demand splitting must refuse. const AVG_CHUNK_SIZE: u64 = 1024; const BLOB_SIZE: usize = 16 * 1024; @@ -1690,7 +1690,7 @@ async fn max_chunk_count_limits_split_and_splice() -> Result<(), Box Store { })) } -// A write whose exact size is >= max_bytes is skipped (drained, never buffered) +// A write whose exact size is >= `max_bytes` is skipped (drained, never buffered) // rather than materialized-then-evicted, and — crucially — it leaves the rest of // the cache untouched (the old buffer-then-evict path would have evicted the // within-budget entry trying to make room for an unstorable blob). @@ -207,7 +207,7 @@ async fn oversized_skip_fires_remove_callbacks() -> Result<(), Error> { } // `MaxSize` is an upper bound, not the real size, so it must NOT trigger the skip: -// a MaxSize over max_bytes whose actual content fits is still cached. +// a `MaxSize` over `max_bytes` whose actual content fits is still cached. #[nativelink_test] async fn max_size_over_budget_with_small_actual_is_stored() -> Result<(), Error> { const DATA: &[u8] = b"ab"; // 2 bytes, well within the 4-byte budget diff --git a/nativelink-util/src/telemetry.rs b/nativelink-util/src/telemetry.rs index 527d8473d..d0de0d0e5 100644 --- a/nativelink-util/src/telemetry.rs +++ b/nativelink-util/src/telemetry.rs @@ -118,7 +118,7 @@ pub async fn init_tracing() -> Result<(), nativelink_error::Error> { // We currently use a UUIDv4 for "service.instance.id" as per: // https://opentelemetry.io/docs/specs/semconv/attributes-registry/service/ - // This might change as we get a better understanding of its usecases in the + // This might change as we get a better understanding of its use cases in the // context of broader observability infrastructure. let resource = Resource::builder() .with_service_name(NATIVELINK_SERVICE_NAME) @@ -246,8 +246,7 @@ pub async fn maybe_load_balanced_channel() -> Option { } } /// This is the header that bazel sends when using the `--remote_header` flag. -/// TODO(palfrey): There are various other headers that bazel supports. -/// Optimize their usage. +/// TODO(palfrey): Bazel supports other headers, and we should optimize their usage. const BAZEL_REQUESTMETADATA_HEADER: &str = "build.bazel.remote.execution.v2.requestmetadata-bin"; use opentelemetry::baggage::BaggageExt; @@ -297,7 +296,7 @@ where fn call(&mut self, req: hyper::http::Request) -> Self::Future { // We must take the current `inner` and not the clone. - // See: https://docs.rs/tower/latest/tower/trait.Service.html#be-careful-when-cloning-inner-services + // See: let clone = self.inner.clone(); let mut inner = core::mem::replace(&mut self.inner, clone); From f871377c09ed6fea7538d372e3a3cd2240c231e2 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Thu, 16 Jul 2026 11:54:42 +0100 Subject: [PATCH 03/84] docs(config-reference): regenerate for NativeLink v1.6.0 (#2553) Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> --- .../reference/nativelink-config/v1.6.0.mdx | 1511 +++++++++++++++++ 1 file changed, 1511 insertions(+) create mode 100644 web/apps/docs/content/docs/reference/nativelink-config/v1.6.0.mdx diff --git a/web/apps/docs/content/docs/reference/nativelink-config/v1.6.0.mdx b/web/apps/docs/content/docs/reference/nativelink-config/v1.6.0.mdx new file mode 100644 index 000000000..4c3bd0f6e --- /dev/null +++ b/web/apps/docs/content/docs/reference/nativelink-config/v1.6.0.mdx @@ -0,0 +1,1511 @@ +--- +title: Configuration reference +description: Every knob in the NativeLink JSON5 configuration — types, defaults, and links to source, autogenerated from the Rust config crate. +full: true +--- + +{/* AUTOGENERATED — do not edit by hand. + Source: nativelink-config @ v1.6.0 (05c5fac5) + Regenerate from web/: bun --filter @nativelink/docs gen:config-reference */} + + + +This is the canonical NativeLink configuration reference for **v1.6.0**. +It is autogenerated from the Rust config crate +([`nativelink-config/src`](https://github.com/TraceMachina/nativelink/tree/v1.6.0/nativelink-config/src)) via the `build-schema` binary, so +it can never drift from what the binary actually deserializes. + +## Top-level fields + +The root object (`CasConfig`) accepts the following fields: + +| Field | Type | Required | Description | +| --- | --- | --- | --- | +| `stores` | array of [NamedStoreConfig](#namedstoreconfig) | Yes | List of stores available to use in this config. The keys can be used in other configs when needing to reference a store. | +| `workers` | array of [WorkerConfig](#workerconfig) | — | Worker configurations used to execute jobs. | +| `schedulers` | array of [NamedSchedulerConfig](#namedschedulerconfig) | — | List of schedulers available to use in this config. The keys can be used in other configs when needing to reference a scheduler. | +| `servers` | array of [ServerConfig](#serverconfig) | Yes | Servers to setup for this process. | +| `experimental_origin_events` | [OriginEventsSpec](#origineventsspec) | — | Experimental - Origin events configuration. This is the service that will collect and publish nativelink events to a store for processing by an external service. | +| `global` | [GlobalConfig](#globalconfig) | — | Any global configurations that apply to all modules live here. | + +## Configuration types + +Every type reachable from the root configuration, in reading order. + +## NamedStoreConfig + +**Common fields** + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `name` | string | Yes | — | | + +Plus exactly one of the following variants (the key selects the variant): + +### `cache_metrics` + +Cache metrics store wraps another store and emits low-cardinality +OpenTelemetry cache operation metrics for the wrapped store. + +This wrapper is opt-in. Stores that are not explicitly wrapped by +`cache_metrics` are constructed exactly as they are without this +wrapper and do not pay its hot-path timing or recording cost. + +**Example JSON5 config:** +```json5 +"cache_metrics": { + "cache_type": "cas", + "backend": { + "filesystem": { + "content_path": "~/.cache/nativelink/content_path-cas", + "temp_path": "~/.cache/nativelink/tmp_path-cas" + } + } +} +``` + +**Type:** [CacheMetricsSpec](#cachemetricsspec) + +### `memory` + +Memory store will store all data in a hash map in memory. + +**Example JSON5 config:** +```json5 +"memory": { + "eviction_policy": { + "max_bytes": "10mb", + } +} +``` + +**Type:** [MemorySpec](#memoryspec) + +### `experimental_cloud_object_store` + +A generic blob store that will store files on the cloud +provider. This configuration will never delete files, so you are +responsible for purging old files in other ways. +It supports the following backends: + +1. **Amazon S3:** + S3 store will use Amazon's S3 service as a backend to store + the files. This configuration can be used to share files + across multiple instances. Uses system certificates for TLS + verification via `rustls-platform-verifier`. + + **Example JSON5 config:** + ```json5 + "experimental_cloud_object_store": { + "provider": "aws", + "region": "eu-north-1", + "bucket": "crossplane-bucket-af79aeca9", + "key_prefix": "test-prefix-index/", + "retry": { + "max_retries": 6, + "delay": 0.3, + "jitter": 0.5 + }, + "multipart_max_concurrent_uploads": 10 + } + ``` + +2. **Google Cloud Storage:** + GCS store uses Google's GCS service as a backend to store + the files. This configuration can be used to share files + across multiple instances. + + **Example JSON5 config:** + ```json5 + "experimental_cloud_object_store": { + "provider": "gcs", + "bucket": "test-bucket", + "key_prefix": "test-prefix-index/", + "retry": { + "max_retries": 6, + "delay": 0.3, + "jitter": 0.5 + }, + "multipart_max_concurrent_uploads": 10 + } + ``` + +3. **Azure Blob Store:** + Azure Blob store will use Microsoft's Azure Blob service as a + backend to store the files. This configuration can be used to + share files across multiple instances. + + **Example JSON5 config:** + ```json5 + "experimental_cloud_object_store": { + "provider": "azure", + "account_name": "cloudshell1393657559", + "container": "simple-test-container", + "key_prefix": "folder/", + "retry": { + "max_retries": 6, + "delay": 0.3, + "jitter": 0.5 + }, + "multipart_max_concurrent_uploads": 10 + } + ``` + +4. **`NetApp` ONTAP S3** + `NetApp` ONTAP S3 store will use ONTAP's S3-compatible storage as a backend + to store files. This store is specifically configured for ONTAP's S3 requirements + including custom TLS configuration, credentials management, and proper vserver + configuration. + + This store uses AWS environment variables for credentials: + - `AWS_ACCESS_KEY_ID` + - `AWS_SECRET_ACCESS_KEY` + - `AWS_DEFAULT_REGION` + + **Example JSON5 config:** + ```json5 + "experimental_cloud_object_store": { + "provider": "ontap", + "endpoint": "https://ontap-s3-endpoint:443", + "vserver_name": "your-vserver", + "bucket": "your-bucket", + "root_certificates": "/path/to/certs.pem", // Optional + "key_prefix": "test-prefix/", // Optional + "retry": { + "max_retries": 6, + "delay": 0.3, + "jitter": 0.5 + }, + "multipart_max_concurrent_uploads": 10 + } + ``` + +5. **Cloudflare R2:** + R2 store uses Cloudflare's R2 service as a backend. R2 speaks the + S3 API, so this is a thin wrapper that derives the account-scoped + endpoint (`https://{account_id}.r2.cloudflarestorage.com`) for you. + + **Example JSON5 config:** + ```json5 + "experimental_cloud_object_store": { + "provider": "r2", + "account_id": "a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4", + "bucket": "nativelink-cas", + "key_prefix": "test-prefix/", + "retry": { + "max_retries": 6, + "delay": 0.3, + "jitter": 0.5 + }, + "multipart_max_concurrent_uploads": 10 + } + ``` + +6. **Oracle Cloud Infrastructure (OCI) Object Storage:** + OCI store uses Oracle Cloud Infrastructure's S3-compatible Object + Storage API. The path-style endpoint is derived from your Object + Storage `namespace` and `region` as + `https://{namespace}.compat.objectstorage.{region}.oci.customer-oci.com`. + Authenticate with a Customer Secret Key (Access Key/Secret Key pair + created under User Settings -> Customer secret keys in the OCI + console); the secret cannot be retrieved after generation, so read + it from an env var via shellexpand. + + **Example JSON5 config:** + ```json5 + "experimental_cloud_object_store": { + "provider": "oci", + "namespace": "your-object-storage-namespace", + "region": "us-phoenix-1", + "bucket": "nativelink-cas", + "access_key_id": "oci_access_key_id", + "secret_access_key": "oci_secret_access_key", + "key_prefix": "test-prefix/", + "retry": { + "max_retries": 6, + "delay": 0.3, + "jitter": 0.5 + } + } + ``` + +**Type:** [ExperimentalCloudObjectSpec](#experimentalcloudobjectspec) + +### `ontap_s3_existence_cache` + +ONTAP S3 Existence Cache provides a caching layer on top of the ONTAP S3 store +to optimize repeated existence checks. It maintains an in-memory cache of object +digests and periodically syncs this cache to disk for persistence. + +The cache helps reduce latency for repeated calls to check object existence, +while still ensuring eventual consistency with the underlying ONTAP S3 store. + +Example JSON5 config: +```json5 +"ontap_s3_existence_cache": { + "index_path": "/path/to/cache/index.json", + "sync_interval_seconds": 300, + "backend": { + "endpoint": "https://ontap-s3-endpoint:443", + "vserver_name": "your-vserver", + "bucket": "your-bucket", + "key_prefix": "test-prefix/" + } +} +``` + +**Type:** [OntapS3ExistenceCacheSpec](#ontaps3existencecachespec) + +### `verify` + +Verify store is used to apply verifications to an underlying +store implementation. It is strongly encouraged to validate +as much data as you can before accepting data from a client, +failing to do so may cause the data in the store to be +populated with invalid data causing all kinds of problems. + +The suggested configuration is to have the CAS validate the +hash and size and the AC validate nothing. + +**Example JSON5 config:** +```json5 +"verify": { + "backend": { + "memory": { + "eviction_policy": { + "max_bytes": "500mb" + } + }, + }, + "verify_size": true, + "verify_hash": true +} +``` + +**Type:** [VerifySpec](#verifyspec) + +### `completeness_checking` + +Completeness checking store verifies if the +output files & folders exist in the CAS before forwarding +the request to the underlying store. +Note: This store should only be used on AC stores. + +**Example JSON5 config:** +```json5 +"completeness_checking": { + "backend": { + "filesystem": { + "content_path": "~/.cache/nativelink/content_path-ac", + "temp_path": "~/.cache/nativelink/tmp_path-ac", + "eviction_policy": { + "max_bytes": "500mb", + } + } + }, + "cas_store": { + "ref_store": { + "name": "CAS_MAIN_STORE" + } + } +} +``` + +**Type:** [CompletenessCheckingSpec](#completenesscheckingspec) + +### `compression` + +A compression store that will compress the data inbound and +outbound. There will be a non-trivial cost to compress and +decompress the data, but in many cases if the final store is +a store that requires network transport and/or storage space +is a concern it is often faster and more efficient to use this +store before those stores. + +**Example JSON5 config:** +```json5 +"compression": { + "compression_algorithm": { + "lz4": {} + }, + "backend": { + "filesystem": { + "content_path": "/tmp/nativelink/data/content_path-cas", + "temp_path": "/tmp/nativelink/data/tmp_path-cas", + "eviction_policy": { + "max_bytes": "2gb", + } + } + } +} +``` + +**Type:** [CompressionSpec](#compressionspec) + +### `dedup` + +A dedup store will take the inputs and run a rolling hash +algorithm on them to slice the input into smaller parts then +run a sha256 algorithm on the slice and if the object doesn't +already exist, upload the slice to the `content_store` using +a new digest of just the slice. Once all parts exist, an +Action-Cache-like digest will be built and uploaded to the +`index_store` which will contain a reference to each +chunk/digest of the uploaded file. Downloading a request will +first grab the index from the `index_store`, and forward the +download content of each chunk as if it were one file. + +This store is exceptionally good when the following conditions +are met: +* Content is mostly the same (inserts, updates, deletes are ok) +* Content is not compressed or encrypted +* Uploading or downloading from `content_store` is the bottleneck. + +Note: This store pairs well when used with `CompressionSpec` as +the `content_store`, but never put `DedupSpec` as the backend of +`CompressionSpec` as it will negate all the gains. + +Note: When running `.has()` on this store, it will only check +to see if the entry exists in the `index_store` and not check +if the individual chunks exist in the `content_store`. + +**Example JSON5 config:** +```json5 +"dedup": { + "index_store": { + "memory": { + "eviction_policy": { + "max_bytes": "1GB", + } + } + }, + "content_store": { + "compression": { + "compression_algorithm": { + "lz4": {} + }, + "backend": { + "fast_slow": { + "fast": { + "memory": { + "eviction_policy": { + "max_bytes": "500MB", + } + } + }, + "slow": { + "filesystem": { + "content_path": "/tmp/nativelink/data/content_path-content", + "temp_path": "/tmp/nativelink/data/tmp_path-content", + "eviction_policy": { + "max_bytes": "2gb" + } + } + } + } + } + } + } +} +``` + +**Type:** [DedupSpec](#dedupspec) + +### `existence_cache` + +Existence store will wrap around another store and cache calls +to has so that subsequent `has_with_results` calls will be +faster. This is useful for cases when you have a store that +is slow to respond to has calls. +Note: This store should only be used on CAS stores. + +**Example JSON5 config:** +```json5 +"existence_cache": { + "backend": { + "memory": { + "eviction_policy": { + "max_bytes": "500mb", + } + } + }, + // Note this is the existence store policy, not the backend policy + "eviction_policy": { + "max_seconds": 100, + } +} +``` + +**Type:** [ExistenceCacheSpec](#existencecachespec) + +### `fast_slow` + +`FastSlow` store will first try to fetch the data from the `fast` +store and then if it does not exist try the `slow` store. +When the object does exist in the `slow` store, it will copy +the data to the `fast` store while returning the data. +This store should be thought of as a store that "buffers" +the data to the `fast` store. +On uploads it will mirror data to both `fast` and `slow` stores. + +WARNING: If you need data to always exist in the `slow` store +for something like remote execution, be careful because this +store will never check to see if the objects exist in the +`slow` store if it exists in the `fast` store (i.e., it assumes +that if an object exists in the `fast` store it will exist in +the `slow` store). + +***Example JSON5 config:*** +```json5 +"fast_slow": { + "fast": { + "filesystem": { + "content_path": "/tmp/nativelink/data/content_path-index", + "temp_path": "/tmp/nativelink/data/tmp_path-index", + "eviction_policy": { + "max_bytes": "500mb", + } + } + }, + "slow": { + "filesystem": { + "content_path": "/tmp/nativelink/data/content_path-index", + "temp_path": "/tmp/nativelink/data/tmp_path-index", + "eviction_policy": { + "max_bytes": "500mb", + } + } + } +} +``` + +**Type:** [FastSlowSpec](#fastslowspec) + +### `shard` + +Shards the data to multiple stores. This is useful for cases +when you want to distribute the load across multiple stores. +The digest hash is used to determine which store to send the +data to. + +**Example JSON5 config:** +```json5 +"shard": { + "stores": [ + { + "store": { + "memory": { + "eviction_policy": { + "max_bytes": "10mb" + }, + }, + }, + "weight": 1 + }] +} +``` + +**Type:** [ShardSpec](#shardspec) + +### `filesystem` + +Stores the data on the filesystem. This store is designed for +local persistent storage. Restarts of this program should restore +the previous state, meaning anything uploaded will be persistent +as long as the filesystem integrity holds. + +**Example JSON5 config:** +```json5 +"filesystem": { + "content_path": "/tmp/nativelink/data-worker-test/content_path-cas", + "temp_path": "/tmp/nativelink/data-worker-test/tmp_path-cas", + "eviction_policy": { + "max_bytes": "10gb", + } +} +``` + +**Type:** [FilesystemSpec](#filesystemspec) + +### `ref_store` + +Store used to reference a store in the root store manager. +This is useful for cases when you want to share a store in different +nested stores. Example, you may want to share the same memory store +used for the action cache, but use a `FastSlowSpec` and have the fast +store also share the memory store for efficiency. + +**Example JSON5 config:** +```json5 +"ref_store": { + "name": "FS_CONTENT_STORE" +} +``` + +**Type:** [RefSpec](#refspec) + +### `size_partitioning` + +Uses the size field of the digest to separate which store to send the +data. This is useful for cases when you'd like to put small objects +in one store and large objects in another store. This should only be +used if the size field is the real size of the content, in other +words, don't use on AC (Action Cache) stores. Any store where you can +safely use `VerifySpec.verify_size = true`, this store should be safe +to use (i.e., CAS stores). + +**Example JSON5 config:** +```json5 +"size_partitioning": { + "size": "128mib", + "lower_store": { + "memory": { + "eviction_policy": { + "max_bytes": "${NATIVELINK_CAS_MEMORY_CONTENT_LIMIT:-100mb}" + } + } + }, + "upper_store": { + /// This store discards data larger than 128mib. + "noop": {} + } +} +``` + +**Type:** [SizePartitioningSpec](#sizepartitioningspec) + +### `grpc` + +This store will pass-through calls to another GRPC store. This store +is not designed to be used as a sub-store of another store, but it +does satisfy the interface and will likely work. + +One major GOTCHA is that some stores use a special function on this +store to get the size of the underlying object, which is only reliable +when this store is serving the a CAS store, not an AC store. If using +this store directly without being a child of any store there are no +side effects and is the most efficient way to use it. + +**Example JSON5 config:** +```json5 +"grpc": { + "instance_name": "main", + "endpoints": [ + {"address": "grpc://${CAS_ENDPOINT:-127.0.0.1}:50051"} + ], + "connections_per_endpoint": "5", + "rpc_timeout_s": "5m", + "store_type": "ac", + // Static headers attached to every outgoing request to the upstream + // remote cache. Useful for fixed service-account credentials. + "headers": { + "authorization": "Bearer my-static-token" + }, + // Header names to copy from the inbound client request and forward to + // the upstream remote cache. Use this to pass through dynamic + // credentials such as a JWT sent by the build client. + "forward_headers": ["authorization", "x-custom-token"] +} +``` + +**Type:** [GrpcSpec](#grpcspec) + +### `redis_store` + +Stores data in any stores compatible with Redis APIs. + +Pairs well with `SizePartitioning` and/or `FastSlow` stores. +Ideal for accepting small object sizes as most Redis store +services have a max file upload of between 256Mb-512Mb. + +**Example JSON5 config:** +```json5 +"redis_store": { + "addresses": [ + "redis://127.0.0.1:6379/", + ], + "max_client_permits": 1000, +} +``` + +**Type:** [RedisSpec](#redisspec) + +### `noop` + +Noop store is a store that sends streams into the void and all data +retrieval will return 404 (`NotFound`). This can be useful for cases +where you may need to partition your data and part of your data needs +to be discarded. + +**Example JSON5 config:** +```json5 +"noop": {} +``` + +**Type:** [NoopSpec](#noopspec) + +### `experimental_mongo` + +Experimental `MongoDB` store implementation. + +This store uses `MongoDB` as a backend for storing data. It supports +both CAS (Content Addressable Storage) and scheduler data with +optional change streams for real-time updates. + +**Example JSON5 config:** +```json5 +"experimental_mongo": { + "connection_string": "mongodb://localhost:27017", + "database": "nativelink", + "cas_collection": "cas", + "key_prefix": "cas:", + "read_chunk_size": 65536, + "max_concurrent_uploads": 10, + "enable_change_streams": false, + "max_requests": "100" +} +``` + +**Type:** [ExperimentalMongoSpec](#experimentalmongospec) + +## WorkerConfig + +Plus exactly one of the following variants (the key selects the variant): + +### `local` + +A worker type that executes jobs locally on this machine. + +**Type:** [LocalWorkerConfig](#localworkerconfig) + +## NamedSchedulerConfig + +**Common fields** + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `name` | string | Yes | — | | + +Plus exactly one of the following variants (the key selects the variant): + +### `simple` + +**Type:** [SimpleSpec](#simplespec) + +### `grpc` + +**Type:** [SchedulerGrpcSpec](#schedulergrpcspec) + +### `cache_lookup` + +**Type:** [CacheLookupSpec](#cachelookupspec) + +### `property_modifier` + +**Type:** [PropertyModifierSpec](#propertymodifierspec) + +### `historical_resource` + +**Type:** [HistoricalResourceSpec](#historicalresourcespec) + +## ServerConfig + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `name` | string | — | {index of server in config} | Name of the server. This is used to help identify the service for telemetry and logs. | +| `listener` | [ListenerConfig](#listenerconfig) | Yes | — | Configuration | +| `services` | [ServicesConfig](#servicesconfig) | — | — | Services to attach to server. | +| `experimental_identity_header` | [IdentityHeaderSpec](#identityheaderspec) | — | {see `IdentityHeaderSpec`} | The config related to identifying the client. | + +## OriginEventsSpec + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `publisher` | [OriginEventsPublisherSpec](#origineventspublisherspec) | Yes | — | The publisher configuration for origin events. | +| `max_event_queue_size` | integer (uint) | — | 65536 (zero defaults to this) | The maximum number of events to queue before applying back pressure. IMPORTANT: Backpressure causes all clients to slow down significantly. Zero is default. | + +## GlobalConfig + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `max_open_files` | integer (uint) | Yes | 24576 (= 24 * 1024) | Maximum number of open files that can be opened at one time. This value is not strictly enforced, it is a best effort. Some internal libraries open files or read metadata from a files which do not obey this limit, however the vast majority of cases will have this limit be honored. This value must be larger than `ulimit -n` to have any effect. Any network open file descriptors is not counted in this limit, but is counted in the kernel limit. It is a good idea to set a very large `ulimit -n`. Note: This value must be greater than 10. | +| `default_digest_hash_function` | [ConfigDigestHashFunction](#configdigesthashfunction) | — | `ConfigDigestHashFunction::sha256` | Default hash function to use while uploading blobs to the CAS when not set by client. | +| `default_digest_size_health_check` | integer (uint) | — | 1024*1024 (1MiB) | Default digest size to use for health check when running diagnostics checks. Health checks are expected to use this size for filling a buffer that is used for creation of digest. | + +## CacheMetricsSpec + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `cache_type` | string | Yes | — | Low-cardinality cache type label for metrics, for example `cas` or `ac`. | +| `backend` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | Store to wrap with cache operation metrics. | + +## MemorySpec + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `eviction_policy` | [EvictionPolicy](#evictionpolicy) | — | — | Policy used to evict items out of the store. Failure to set this value will cause items to never be removed from the store causing infinite memory usage. | + +## ExperimentalCloudObjectSpec + +See [`experimental_cloud_object_store`](#experimental_cloud_object_store-1) for details + +## OntapS3ExistenceCacheSpec + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `index_path` | string | Yes | — | | +| `sync_interval_seconds` | integer (uint32) | Yes | — | | +| `backend` | [ExperimentalOntapS3Spec](#experimentalontaps3spec) | Yes | — | | + +## VerifySpec + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `backend` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | The underlying store wrap around. All content will first flow through self before forwarding to backend. In the event there is an error detected in self, the connection to the backend will be terminated, and early termination should always cause updates to fail on the backend. | +| `verify_size` | boolean | — | `false` | If set the store will verify the size of the data before accepting an upload of data. | +| `verify_hash` | boolean | — | `false` | If the data should be hashed and verify that the key matches the computed hash. The hash function is automatically determined based request and if not set will use the global default. | + +## CompletenessCheckingSpec + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `backend` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | The underlying store that will have it's results validated before sending to client. | +| `cas_store` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | When a request is made, the results are decoded and all output digests/files are verified to exist in this CAS store before returning success. | + +## CompressionSpec + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `backend` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | The underlying store wrap around. All content will first flow through self before forwarding to backend. In the event there is an error detected in self, the connection to the backend will be terminated, and early termination should always cause updates to fail on the backend. | +| `compression_algorithm` | [CompressionAlgorithm](#compressionalgorithm) | Yes | — | The compression algorithm to use. | + +## DedupSpec + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `index_store` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | Store used to store the index of each dedup slice. This store should generally be fast and small. | +| `content_store` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | The store where the individual chunks will be uploaded. This store should generally be the slower & larger store. | +| `min_size` | integer (uint32) | — | 64k | Minimum size that a chunk will be when slicing up the content. Note: This setting can be increased to improve performance because it will actually not check this number of bytes when deciding where to partition the data. | +| `normal_size` | integer (uint32) | — | 256k | A best-effort attempt will be made to keep the average size of the chunks to this number. It is not a guarantee, but a slight attempt will be made. | +| `max_size` | integer (uint32) | — | 512k | Maximum size a chunk is allowed to be. | +| `max_concurrent_fetch_per_get` | integer (uint32) | — | 10 | Due to implementation detail, we want to prefer to download the first chunks of the file so we can stream the content out and free up some of our buffers. This configuration will be used to restrict the number of concurrent chunk downloads at a time per `get()` request. | + +## ExistenceCacheSpec + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `backend` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | The underlying store wrap around. All content will first flow through self before forwarding to backend. In the event there is an error detected in self, the connection to the backend will be terminated, and early termination should always cause updates to fail on the backend. | +| `eviction_policy` | [EvictionPolicy](#evictionpolicy) | — | — | Policy used to evict items out of the store. Failure to set this value will cause items to never be removed from the store causing infinite memory usage. | + +## FastSlowSpec + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `fast` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | Fast store that will be attempted to be contacted before reaching out to the `slow` store. | +| `fast_direction` | [StoreDirection](#storedirection) | — | `"both"` | How to handle the fast store. This can be useful to set to Get for worker nodes such that results are persisted to the slow store only. | +| `slow` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | If the object does not exist in the `fast` store it will try to get it from this store. | +| `slow_direction` | [StoreDirection](#storedirection) | — | `"both"` | How to handle the slow store. This can be useful if creating a diode and you wish to have an upstream read only store. | +| `bypass_dedup_threshold_bytes` | integer (uint64) | — | disabled (0) | Reads of blobs at or above this size skip the leader/follower dedup map and stream straight from the slow store without populating the fast tier. `0` (the default) disables the bypass: every read goes through dedup, matching the prior behaviour. Enable it by setting a threshold — 256 MiB is a reasonable starting point for backends where large-blob dedup is a net loss (followers tend to time out anyway), but the right value is workload-dependent. | + +## ShardSpec + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `stores` | array of [ShardConfig](#shardconfig) | Yes | — | Stores to shard the data to. | + +## FilesystemSpec + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `content_path` | string | Yes | — | Path on the system where to store the actual content. This is where the bulk of the data will be placed. On service startup this folder will be scanned and all files will be added to the cache. In the event one of the files doesn't match the criteria, the file will be deleted. | +| `temp_path` | string | Yes | — | A temporary location of where files that are being uploaded or deleted will be placed while the content cannot be guaranteed to be accurate. This location must be on the same block device as `content_path` so atomic moves can happen (i.e., move without copy). All files in this folder will be deleted on every startup. | +| `read_buffer_size` | integer (uint32) | — | 32k | Buffer size to use when reading files. Generally this should be left to the default value except for testing. | +| `eviction_policy` | [EvictionPolicy](#evictionpolicy) | — | — | Policy used to evict items out of the store. Failure to set this value will cause items to never be removed from the store causing infinite memory usage. | +| `block_size` | integer (uint64) | — | 4kb | The block size of the filesystem for the running machine value is used to determine an entry's actual size on disk consumed For a 4KB block size filesystem, a 1B file actually consumes 4KB | +| `max_concurrent_writes` | integer (uint) | — | unlimited | Maximum number of concurrent write operations allowed. Each write involves streaming data to a temp file and calling `sync_all()`, which can saturate disk I/O when many writes happen simultaneously. Limiting concurrency prevents disk saturation from blocking the async runtime. A value of 0 means unlimited (no concurrency limit). | + +## RefSpec + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `name` | string | Yes | — | Name of the store under the root "stores" config object. | + +## SizePartitioningSpec + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `size` | integer (uint64) | Yes | — | Size to partition the data on. | +| `lower_store` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | Store to send data when object is < (less than) size. | +| `upper_store` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | Store to send data when object is >= (less than eq) size. | + +## GrpcSpec + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `instance_name` | string | — | `""` | Instance name for GRPC calls. Proxy calls will have the `instance_name` changed to this. | +| `endpoints` | array of [GrpcEndpoint](#grpcendpoint) | Yes | — | The endpoint of the grpc connection. | +| `store_type` | [StoreType](#storetype) | Yes | — | The type of the upstream store, this ensures that the correct server calls are made. | +| `retry` | [Retry](#retry) | — | — | Retry configuration to use when a network request fails. | +| `max_concurrent_requests` | integer (uint) | — | `0` | Limit the number of simultaneous upstream requests to this many. A value of zero is treated as unlimited. If the limit is reached the request is queued. | +| `connections_per_endpoint` | integer (uint) | — | `0` | The number of connections to make to each specified endpoint to balance the load over multiple TCP connections. Default 1. | +| `rpc_timeout_s` | integer (uint64) | — | 0 (disabled) | Maximum time (seconds) allowed for a single RPC request (e.g. a `ByteStream.Write` call) before it is cancelled. | +| `use_legacy_resource_names` | boolean | — | false | Use legacy `ByteStream` resource name format, omitting the digest function component from the path. | +| `headers` | map of string to string | — | — | Static headers to attach to every outgoing gRPC request sent to this store's upstream endpoints. Useful for fixed authentication tokens (e.g. `{"authorization": "Bearer "}`) and other static metadata. | +| `forward_headers` | array of string | — | — | Header names to forward from the incoming client request to every outgoing upstream request. The header value is taken from the client request that triggered this store operation. Use this to pass through dynamic credentials such as JWT tokens sent by build clients. | + +## RedisSpec + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `addresses` | array of string | Yes | — | The hostname or IP address of the Redis server. Ex: `["redis://username:password@redis-server-url:6380/99"]` 99 Represents database ID, 6380 represents the port. | +| `response_timeout_s` | integer (uint64) | — | 10 | DEPRECATED: use `command_timeout_ms` The response timeout for the Redis connection in seconds. | +| `connection_timeout_s` | integer (uint64) | — | 10 | DEPRECATED: use `connection_timeout_ms` | +| `experimental_pub_sub_channel` | string | — | (Empty String / No Channel) | An optional and experimental Redis channel to publish write events to. | +| `key_prefix` | string | — | (Empty String / No Prefix) | An optional prefix to prepend to all keys in this store. | +| `mode` | [RedisMode](#redismode) | — | standard, | Set the mode Redis is operating in. | +| `broadcast_channel_capacity` | integer (uint) | — | `0` | Deprecated as redis-rs doesn't use it | +| `command_timeout_ms` | integer (uint64) | — | 10000 (10 seconds) | The amount of time in milliseconds until the Redis store considers the command to be timed out. This will trigger a retry of the command and potentially a reconnection to the Redis server. | +| `connection_timeout_ms` | integer (uint64) | — | 3000 (3 seconds) | The amount of time in milliseconds until the Redis store considers the connection to unresponsive. This will trigger a reconnection to the Redis server. | +| `health_check_timeout_ms` | integer (uint64) | — | 4000 (4 seconds) | Per-call ceiling for the `check_health` PING in milliseconds. | +| `read_chunk_size` | integer (uint) | — | 64KiB | The amount of data to read from the Redis server at a time. This is used to limit the amount of memory used when reading large objects from the Redis server as well as limiting the amount of time a single read operation can take. | +| `connection_pool_size` | integer (uint) | — | 3 | The number of connections to keep open to the Redis servers. | +| `max_chunk_uploads_per_update` | integer (uint) | — | 10 | The maximum number of upload chunks to allow per update. This is used to limit the amount of memory used when uploading large objects to the Redis server. A good rule of thumb is to think of the data as: `AVAIL_MEMORY / (read_chunk_size * max_chunk_uploads_per_update) = THORETICAL_MAX_CONCURRENT_UPLOADS` (note: it is a good idea to divide `AVAIL_MAX_MEMORY` by ~10 to account for other memory usage) | +| `scan_count` | integer (uint) | — | 10000 | The COUNT value passed when scanning keys in Redis. This is used to hint the amount of work that should be done per response. | +| `retry` | [Retry](#retry) | — | — | Retry configuration to use when a network request fails. | +| `max_client_permits` | integer (uint) | — | 500 | Maximum number of permitted actions to the Redis store at any one time This stops problems with timeouts due to many, many inflight actions | +| `max_count_per_cursor` | integer (uint64) | — | 1500 | Maximum number of items returned per cursor for the search indexes May reduce thundering herd issues with worker provisioner at higher node counts, | + +## NoopSpec + +_No fields._ + +## ExperimentalMongoSpec + +Configuration for `ExperimentalMongoDB` store. + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `connection_string` | string | Yes | — | `ExperimentalMongoDB` connection string. Example: <mongodb://localhost:27017> or <mongodb+srv://cluster.mongodb.net> | +| `database` | string | — | "nativelink" | The database name to use. | +| `cas_collection` | string | — | "cas" | The collection name for CAS data. | +| `scheduler_collection` | string | — | "scheduler" | The collection name for scheduler data. | +| `key_prefix` | string | — | "" | Prefix to prepend to all keys stored in `MongoDB`. | +| `read_chunk_size` | integer (uint) | — | 65536 (64KB) | The maximum amount of data to read from `MongoDB` in a single chunk (in bytes). | +| `max_concurrent_uploads` | integer (uint) | — | 10 | Deprecated, unused Maximum number of concurrent uploads allowed. | +| `connection_timeout_ms` | integer (uint64) | — | 3000 | Connection timeout in milliseconds. | +| `command_timeout_ms` | integer (uint64) | — | 10000 | Command timeout in milliseconds. | +| `enable_change_streams` | boolean | — | false | Enable `MongoDB` change streams for real-time updates. Required for scheduler subscriptions. | +| `write_concern_w` | string | — | — | Write concern 'w' parameter. Can be a number (e.g., 1) or string (e.g., "majority"). | +| `write_concern_j` | boolean | — | — | Write concern 'j' parameter (journal acknowledgment). | +| `write_concern_timeout_ms` | integer (uint32) | — | — | Write concern timeout in milliseconds. | +| `max_requests` | integer (uint) | — | Unlimited | Limits the number of requests at any one time | + +## LocalWorkerConfig + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `name` | string | — | {Index position in the workers list} | Name of the worker. This is give a more friendly name to a worker for logging and metric publishing. This is also the prefix of the worker id (i.e., "{name}{uuidv6}"). | +| `worker_api_endpoint` | [EndpointConfig](#endpointconfig) | Yes | — | Endpoint which the worker will connect to the scheduler's `WorkerApiService`. | +| `max_action_timeout_s` | integer (uint) | — | 20 minutes | The maximum time an action is allowed to run. If a task requests for a timeout longer than this time limit, the task will be rejected. Value in seconds. | +| `max_upload_timeout_s` | integer (uint) | — | 10 minutes | Maximum time allowed for uploading action results to CAS after execution completes. If upload takes longer than this, the action fails with `DeadlineExceeded` and may be retried by the scheduler. Value in seconds. | +| `max_cleanup_wait_s` | integer (uint) | — | 30 seconds | Maximum time to wait for action directory cleanup before timing out. Value in seconds. | +| `max_cleanup_backoff_ms` | integer (uint) | — | 500 milliseconds | Maximum backoff duration for exponential backoff when waiting for cleanup. Value in milliseconds. | +| `max_inflight_tasks` | integer (uint64) | — | 0 (infinite tasks) | Maximum number of inflight tasks this worker can cope with. | +| `timeout_handled_externally` | boolean | — | false (`NativeLink` fully handles timeouts) | If timeout is handled in `entrypoint` or another wrapper script. If set to true `NativeLink` will not honor the timeout the action requested and instead will always force kill the action after `max_action_timeout` has been reached. If this is set to false, the smaller value of the action's timeout and `max_action_timeout` will be used to which `NativeLink` will kill the action. | +| `entrypoint` | string | — | {Use the command from the job request} | The command to execute on every execution request. This will be parsed as a command + arguments (not shell). Example: "run.sh" and a job with command: "sleep 5" will result in a command like: "run.sh sleep 5". | +| `experimental_precondition_script` | string | — | — | An optional script to run before every action is processed on the worker. The value should be the full path to the script to execute and will pause all actions on the worker if it returns an exit code other than 0. If not set, then the worker will never pause and will continue to accept jobs according to the scheduler configuration. This is useful, for example, if the worker should not take any more actions until there is enough resource available on the machine to handle them. | +| `cas_fast_slow_store` | string | Yes | — | Underlying CAS store that the worker will use to download CAS artifacts. This store must be a `FastSlowStore`. The `fast` store must be a `FileSystemStore` because it will use hardlinks when building out the files instead of copying the files. The slow store must eventually resolve to the same store the scheduler/client uses to send job requests. | +| `upload_action_result` | [UploadActionResultConfig](#uploadactionresultconfig) | — | — | Configuration for uploading action results. | +| `work_directory` | string | Yes | — | The directory work jobs will be executed from. This directory will be fully managed by the worker service and will be purged on startup. This directory and the directory referenced in `local_filesystem_store_ref`'s `stores::FilesystemStore::content_path` must be on the same filesystem. Hardlinks will be used when placing files that are accessible to the jobs that are sourced from `local_filesystem_store_ref`'s `content_path`. | +| `platform_properties` | map of string to [WorkerProperty](#workerproperty) | Yes | — | Properties of this worker. This configuration will be sent to the scheduler and used to tell the scheduler to restrict what should be executed on this worker. | +| `additional_environment` | map of string to [EnvironmentSource](#environmentsource) | — | — | An optional mapping of environment names to set for the execution as well as those specified in the action itself. If set, will set each key as an environment variable before executing the job with the value of the environment variable being the value of the property of the action being executed of that name or the fixed value. | +| `directory_cache` | [DirectoryCacheConfig](#directorycacheconfig) | — | — | Optional directory cache configuration for improving performance by caching reconstructed input directories and using hardlinks instead of rebuilding them from CAS for every action. | +| `use_namespaces` | boolean | — | False | Whether to use namespaces to isolate the execution. This is only available on Linux. It is highly recommended as it avoids a number of issues with zombie processes and also provides additional hermeticity. If explicitly set to true and it is not supported the worker will exit with an error. | +| `use_mount_namespace` | boolean | — | False | Whether to use a mount namespace to isolate the worker root. This is only available on Linux and when `use_namespaces` is true. It is highly recommended provides additional hermeticity. If explicitly set to true and it is not supported or `use_namespaces` is not set to true the worker will exit with an error. | + +## SimpleSpec + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `supported_platform_properties` | map of string to [PropertyType](#propertytype) | — | — | A list of supported platform properties mapped to how these properties are used when the scheduler looks for worker nodes capable of running the task. | +| `retain_completed_for_s` | integer (uint32) | — | 60 seconds | The amount of time to retain completed actions for in case a `WaitExecution` is called after the action has completed. | +| `client_action_timeout_s` | integer (uint64) | — | 60 seconds | Mark operations as completed with error if no client has updated them within this duration. | +| `worker_timeout_s` | integer (uint64) | — | 5 seconds | Remove workers from pool once the worker has not responded in this amount of time in seconds. | +| `max_action_executing_timeout_s` | integer (uint64) | — | 0 (disabled) | Maximum time (seconds) an action can stay in Executing state without any worker update before being timed out and re-queued. This applies regardless of worker keepalive status, catching cases where a worker is alive (sending keepalives) but stuck on a specific action. Set to 0 to disable (relies only on `worker_timeout_s`). | +| `max_job_retries` | integer (uint) | — | 3 | If a job returns an internal error or times out this many times when attempting to run on a worker the scheduler will return the last error to the client. Jobs will be retried and this configuration is to help prevent one rogue job from infinitely retrying and taking up a lot of resources when the task itself is the one causing the server to go into a bad state. | +| `allocation_strategy` | [WorkerAllocationStrategy](#workerallocationstrategy) | — | `"least_recently_used"` | The strategy used to assign workers jobs. | +| `experimental_backend` | [ExperimentalSimpleSchedulerBackend](#experimentalsimpleschedulerbackend) | — | memory | The storage backend to use for the scheduler. | +| `worker_match_logging_interval_s` | integer (int64) | — | `10` | Every N seconds, do logging of worker matching e.g. "worker busy", "can't find any worker" Defaults to 10s. Can be set to `-1` to disable | + +## SchedulerGrpcSpec + +A scheduler that forwards requests to an upstream scheduler. This +is useful to use when doing some kind of local action cache or CAS away from +the main cluster of workers. In general, it's more efficient to point the +build at the main scheduler directly though. + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `endpoint` | [GrpcEndpoint](#grpcendpoint) | Yes | — | The upstream scheduler to forward requests to. | +| `retry` | [Retry](#retry) | — | — | Retry configuration to use when a network request fails. | +| `max_concurrent_requests` | integer (uint) | — | unlimited | Limit the number of simultaneous upstream requests to this many. A value of zero is treated as unlimited. If the limit is reached the request is queued. | +| `connections_per_endpoint` | integer (uint) | — | 1 | The number of connections to make to each specified endpoint to balance the load over multiple TCP connections. | + +## CacheLookupSpec + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `ac_store` | string | Yes | — | The reference to the action cache store used to return cached actions from rather than running them again. To prevent unintended issues, this store should probably be a `CompletenessCheckingSpec`. | +| `scheduler` | [SchedulerSpec](#schedulerspec) | Yes | — | The nested scheduler to use if cache lookup fails. | + +## PropertyModifierSpec + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `modifications` | array of [PropertyModification](#propertymodification) | Yes | — | A list of modifications to perform to incoming actions for the nested scheduler. These are performed in order and blindly, so removing a property that doesn't exist is fine and overwriting an existing property is also fine. If adding properties that do not exist in the nested scheduler is not supported and will likely cause unexpected behaviour. | +| `scheduler` | [SchedulerSpec](#schedulerspec) | Yes | — | The nested scheduler to use after modifying the properties. | + +## HistoricalResourceSpec + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `hints_file` | string | Yes | — | JSON file containing historical resource hints keyed by Bazel `RequestMetadata` `target_id` and/or `action_mnemonic`. | +| `refresh_interval_s` | integer (uint64) | — | 30 seconds | Reload interval for `hints_file`. Set to 0 to load once. | +| `cpu_property_name` | string | — | `cpu_count` | Platform property name used for CPU minimum values. | +| `memory_property_name` | string | — | `memory_kb` | Platform property name used for memory minimum values, expressed in KiB. | +| `scheduler` | [SchedulerSpec](#schedulerspec) | Yes | — | The nested scheduler to use after applying resource hints. | + +## ListenerConfig + +Plus exactly one of the following variants (the key selects the variant): + +### `http` + +Listener for HTTP/HTTPS/HTTP2 sockets. + +**Type:** [HttpListener](#httplistener) + +## ServicesConfig + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `cas` | array of [CasServiceConfig](#casserviceconfig) | — | — | The Content Addressable Storage (CAS) backend config. The key is the `instance_name` used in the protocol and the value is the underlying CAS store config. | +| `ac` | array of [ActionCacheServiceConfig](#actioncacheserviceconfig) | — | — | The Action Cache (AC) backend config. The key is the `instance_name` used in the protocol and the value is the underlying AC store config. | +| `capabilities` | array of [CapabilitiesServiceConfig](#capabilitiesserviceconfig) | — | — | Capabilities service is required in order to use most of the bazel protocol. This service is used to provide the supported features and versions of this bazel GRPC service. | +| `execution` | array of [ExecutionServiceConfig](#executionserviceconfig) | — | — | The remote execution service configuration. NOTE: This service is under development and is currently just a place holder. | +| `bytestream` | array of [ByteStreamServiceConfig](#bytestreamserviceconfig) | — | — | This is the service used to stream data to and from the CAS. Bazel's protocol strongly encourages users to use this streaming interface to interact with the CAS when the data is large. | +| `fetch` | array of [FetchServiceConfig](#fetchserviceconfig) | — | — | These two are collectively the Remote Asset protocol, but it's defined as two separate services | +| `push` | array of [PushServiceConfig](#pushserviceconfig) | — | — | | +| `worker_api` | [WorkerApiConfig](#workerapiconfig) | — | — | This is the service used for workers to connect and communicate through. NOTE: This service should be served on a different, non-public port. In other words, `worker_api` configuration should not have any other services that are served on the same port. Doing so is a security risk, as workers have a different permission set than a client that makes the remote execution/cache requests. | +| `experimental_bep` | [BepConfig](#bepconfig) | — | — | Experimental - Build Event Protocol (BEP) configuration. This is the service that will consume build events from the client and publish them to a store for processing by an external service. | +| `admin` | [AdminConfig](#adminconfig) | — | — | This is the service for any administrative tasks. It provides a REST API endpoint for administrative purposes. | +| `health` | [HealthConfig](#healthconfig) | — | — | This is the service for health status check. | + +## IdentityHeaderSpec + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `header_name` | string | — | "x-identity" | The name of the header to look for the identity in. | +| `required` | boolean | — | `false` | If the header is required to be set or fail the request. | + +## OriginEventsPublisherSpec + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `store` | string | Yes | — | The store to publish nativelink events to. The store name referenced in the `stores` map in the main config. | + +## ConfigDigestHashFunction + +| Value | Description | +| --- | --- | +| `"sha256"` | Use the sha256 hash function. [https://en.wikipedia.org/wiki/SHA-2](https://en.wikipedia.org/wiki/SHA-2) | +| `"blake3"` | Use the blake3 hash function. [https://en.wikipedia.org/wiki/BLAKE_(hash_function)](https://en.wikipedia.org/wiki/BLAKE_(hash_function)) | + +## EvictionPolicy + +Eviction policy always works on LRU (Least Recently Used). Any time an entry +is touched it updates the timestamp. Inserts and updates will execute the +eviction policy removing any expired entries and/or the oldest entries +until the store size becomes smaller than `max_bytes`. + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `max_bytes` | integer (uint) | — | 0 | Maximum number of bytes before eviction takes place. Zero means never evict based on size. | +| `evict_bytes` | integer (uint) | — | 0 | When eviction starts based on hitting `max_bytes`, continue until `max_bytes - evict_bytes` is met to create a low watermark. This stops operations from thrashing when the store is close to the limit. | +| `max_seconds` | integer (uint32) | — | 0 | Maximum number of seconds for an entry to live since it was last accessed before it is evicted. Zero means never evict based on time. | +| `max_count` | integer (uint64) | — | 0 | Maximum size of the store before an eviction takes place. Zero means never evict based on count. | + +## Retry + +Retry configuration. This configuration is exponential and each iteration +a jitter as a percentage is applied of the calculated delay. For example: +```haskell +Retry{ + max_retries: 7, + delay: 0.1, + jitter: 0.5, +} +``` +will result in: + +| Attempt | Delay | +| --- | --- | +| 1 | 0 ms | +| 2 | 75 to 125 ms | +| 3 | 150 to 250 ms | +| 4 | 300 to 500 ms | +| 5 | 600 ms to 1 s | +| 6 | 1.2 to 2 s | +| 7 | 2.4 to 4 s | +| 8 | 4.8 to 8 s | + +The total delay is additive, so this example produces 9.525 to 15.875 s of total delay for a single request. + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `max_retries` | integer (uint) | — | `0` | Maximum number of retries until retrying stops. Setting this to zero will always attempt 1 time, but not retry. | +| `delay` | number (float) | — | `0` | Delay in seconds for exponential back off. | +| `jitter` | number (float) | — | `0` | Amount of jitter to add as a percentage in decimal form. This will change the formula like: | +| `retry_on_errors` | array of [ErrorCode](#errorcode) | — | — | A list of error codes to retry on, if this is not set then the default error codes to retry on are used. These default codes are the most likely to be non-permanent: `Unknown`; `Cancelled`; `DeadlineExceeded`; `ResourceExhausted`; `Aborted`; `Internal`; `Unavailable`; `DataLoss` | + +## ExperimentalOntapS3Spec + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `endpoint` | string | Yes | — | | +| `vserver_name` | string | Yes | — | | +| `bucket` | string | Yes | — | | +| `root_certificates` | string | — | — | | +| `key_prefix` | string | — | — | If you wish to prefix the location in the bucket. If None, no prefix will be used. | +| `retry` | [Retry](#retry) | — | — | Retry configuration to use when a network request fails. | +| `consider_expired_after_s` | integer (uint32) | — | 0 | If the number of seconds since the `last_modified` time of the object is greater than this value, the object will not be considered "existing". This allows for external tools to delete objects that have not been uploaded in a long time. If a client receives a `NotFound` the client should re-upload the object. | +| `max_retry_buffer_per_request` | integer (uint) | — | 5MB | The maximum buffer size to retain in case of a retryable error during upload. Setting this to zero will disable upload buffering; this means that in the event of a failure during upload, the entire upload will be aborted and the client will likely receive an error. | +| `multipart_max_concurrent_uploads` | integer (uint) | — | 10 | Maximum number of concurrent `UploadPart` requests per `MultipartUpload`. | +| `insecure_allow_http` | boolean | — | false | Allow unencrypted HTTP connections. Only use this for local testing. | +| `disable_http2` | boolean | — | false | Disable HTTP/2 connections and only use HTTP/1.1. Default client configuration will have HTTP/1.1 and HTTP/2 enabled for connection schemes. HTTP/2 should be disabled if environments have poor support or performance related to HTTP/2. Safe to keep default unless underlying network environment, S3, or GCS API servers specify otherwise. | + +## CompressionAlgorithm + +Plus exactly one of the following variants (the key selects the variant): + +### `lz4` + +LZ4 compression algorithm is extremely fast for compression and +decompression, however does not perform very well in compression +ratio. In most cases build artifacts are highly compressible, however +lz4 is quite good at aborting early if the data is not deemed very +compressible. + +see: [https://lz4.github.io/lz4/](https://lz4.github.io/lz4/) + +**Type:** [Lz4Config](#lz4config) + +## StoreDirection + +| Value | Description | +| --- | --- | +| `"both"` | The store operates normally and all get and put operations are handled by it. | +| `"update"` | Update operations will cause persistence to this store, but Get operations will be ignored. This only makes sense on the fast store as the slow store will never get written to on Get anyway. | +| `"get"` | Get operations will cause persistence to this store, but Update operations will be ignored. | +| `"read_only"` | Operate as a read only store, only really makes sense if there's another way to write to it. | + +## ShardConfig + +Configuration for an individual shard of the store. + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `store` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | Store to shard the data to. | +| `weight` | integer (uint32) | — | 1 | The weight of the store. This is used to determine how much data should be sent to the store. The actual percentage is the sum of all the store's weights divided by the individual store's weight. | + +## GrpcEndpoint + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `address` | string | Yes | — | The endpoint address (i.e. grpc://example.com:443 or grpcs://example.com:443). | +| `tls_config` | [ClientTlsConfig](#clienttlsconfig) | — | — | The TLS configuration to use to connect to the endpoint (if grpcs). | +| `concurrency_limit` | integer (uint) | — | — | The maximum concurrency to allow on this endpoint. | +| `connect_timeout_s` | integer (uint64) | — | 30 seconds | Timeout for establishing a TCP connection to the endpoint (seconds). | +| `tcp_keepalive_s` | integer (uint64) | — | 30 seconds | TCP keepalive interval (seconds). Sends TCP keepalive probes at this interval to detect dead connections at the OS level. | +| `http2_keepalive_interval_s` | integer (uint64) | — | 30 seconds | HTTP/2 keepalive interval (seconds). Sends HTTP/2 PING frames at this interval to detect dead connections at the application level. | +| `http2_keepalive_timeout_s` | integer (uint64) | — | 20 seconds | HTTP/2 keepalive timeout (seconds). If a PING response is not received within this duration, the connection is considered dead. | + +## StoreType + +| Value | Description | +| --- | --- | +| `"cas"` | The store is content addressable storage. | +| `"ac"` | The store is an action cache. | + +## RedisMode + +| Value | Description | +| --- | --- | +| `"cluster"` | Use Redis Cluster. | +| `"sentinel"` | Use Redis Sentinel. | +| `"standard"` | Use a standalone Redis server. | + +## EndpointConfig + +Generic config for an endpoint and associated configs. + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `uri` | string | Yes | — | URI of the endpoint. | +| `timeout` | number (float) | — | 5 seconds | Timeout in seconds that a request should take. | +| `tls_config` | [ClientTlsConfig](#clienttlsconfig) | — | — | The TLS configuration to use to connect to the endpoint. | + +## UploadActionResultConfig + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `ac_store` | string | — | {No uploading is done} | Underlying AC store that the worker will use to publish execution results into. Objects placed in this store should be reachable from the scheduler/client-cas after they have finished updating. | +| `upload_ac_results_strategy` | [UploadCacheResultsStrategy](#uploadcacheresultsstrategy) | — | `SuccessOnly` | In which situations should the results be published to the `ac_store`, if set to `SuccessOnly` then only results with an exit code of 0 will be uploaded, if set to Everything all completed results will be uploaded. | +| `historical_results_store` | string | — | {CAS store of parent} | Store to upload historical results to. This should be a CAS store if set. | +| `upload_historical_results_strategy` | [UploadCacheResultsStrategy](#uploadcacheresultsstrategy) | — | `FailuresOnly` | In which situations should the results be published to the historical CAS. The historical CAS is where failures are published. These messages conform to the CAS key-value lookup format and are always a `HistoricalExecuteResponse` serialized message. | +| `success_message_template` | string | — | "" (no message) | Template to use for the `ExecuteResponse.message` property. This message is attached to the response before it is sent to the client. The following special variables are supported:; `digest_function`: Digest function used to calculate the action digest: `action_digest_hash`: Action digest hash: `action_digest_size`: Action digest size: `historical_results_hash`: `HistoricalExecuteResponse` digest hash: `historical_results_size`: `HistoricalExecuteResponse` digest size. | +| `failure_message_template` | string | — | "" (no message) | Same as `success_message_template` but for failure case. | + +## WorkerProperty + +Plus exactly one of the following variants (the key selects the variant): + +### `values` + +List of static values. +Note: Generally there should only ever be 1 value, but if the platform +property key is `PropertyType::Priority` it may have more than one value. + +**Type:** array of string + +### `query_cmd` + +A dynamic configuration. The string will be executed as a command +(not shell) and will be split by "\n" (new line character). + +**Type:** string + +## EnvironmentSource + +One of: + +- `property`: string — The name of the platform property in the action to get the value from. +- `value`: string — The raw value to set. +- `"from_environment"` — Take the value from the local environment corresponding to the name key +- `"timeout_millis"` — The max amount of time in milliseconds the command is allowed to run (requested by the client). +- `"side_channel_file"` — A special file path will be provided that can be used to communicate with the parent process about out-of-band information. This file will be read after the command has finished executing. Based on the contents of the file, the behavior of the result may be modified. +- `"action_directory"` — A "root" directory for the action. This directory can be used to store temporary files that are not needed after the action has completed. This directory will be purged after the action has completed. + +## DirectoryCacheConfig + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `max_entries` | integer (uint) | — | 1000 | Maximum number of cached directories. | +| `max_size_bytes` | integer (uint64) | — | 10737418240 (10 GB) | Maximum total size in bytes for all cached directories (0 = unlimited). | +| `cache_root` | string | — | `{work_directory}/../directory_cache` | Base directory for cache storage. This directory will be managed by the worker and should be on the same filesystem as `work_directory`. | + +## PropertyType + +When the scheduler matches tasks to workers that are capable of running +the task, this value will be used to determine how the property is treated. + +One of: + +- string +- `"minimum"` — Requires the platform property to be a u64 and when the scheduler looks for appropriate worker nodes that are capable of executing the task, the task will not run on a node that has less than this value. +- `"exact"` — Requires the platform property to be a string and when the scheduler looks for appropriate worker nodes that are capable of executing the task, the task will not run on a node that does not have this property set to the value with exact string match. +- `"priority"` — Does not restrict on this value and instead will be passed to the worker as an informational piece. In the future this will be used by the scheduler and worker to cause the scheduler to prefer certain workers over others, but not restrict them based on these values. + +## WorkerAllocationStrategy + +When a worker is being searched for to run a job, this will be used +on how to choose which worker should run the job when multiple +workers are able to run the task. + +| Value | Description | +| --- | --- | +| `"least_recently_used"` | Prefer workers that have been least recently used to run a job. | +| `"most_recently_used"` | Prefer workers that have been most recently used to run a job. | + +## ExperimentalSimpleSchedulerBackend + +One of: + +- `"memory"` — Use an in-memory store for the scheduler. +- `redis`: [ExperimentalRedisSchedulerBackend](#experimentalredisschedulerbackend) — Use a Redis store for the scheduler. + +## SchedulerSpec + +Plus exactly one of the following variants (the key selects the variant): + +### `simple` + +**Type:** [SimpleSpec](#simplespec) + +### `grpc` + +**Type:** [SchedulerGrpcSpec](#schedulergrpcspec) + +### `cache_lookup` + +**Type:** [CacheLookupSpec](#cachelookupspec) + +### `property_modifier` + +**Type:** [PropertyModifierSpec](#propertymodifierspec) + +### `historical_resource` + +**Type:** [HistoricalResourceSpec](#historicalresourcespec) + +## PropertyModification + +Plus exactly one of the following variants (the key selects the variant): + +### `add` + +Add a property to the action properties. + +**Type:** [PlatformPropertyAddition](#platformpropertyaddition) + +### `remove` + +Remove a named property from the action. + +**Type:** string + +### `replace` + +If a property is found, then replace it with another one. + +**Type:** [PlatformPropertyReplacement](#platformpropertyreplacement) + +## HttpListener + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `socket_address` | string | Yes | — | Address to listen on. Example: `127.0.0.1:8080` or `:8080` to listen to all IPs. | +| `freebind` | boolean | — | false | Allow binding `socket_address` before it is assigned locally. | +| `compression` | [HttpCompressionConfig](#httpcompressionconfig) | — | — | Data transport compression configuration to use for this service. | +| `advanced_http` | [HttpServerConfig](#httpserverconfig) | — | — | Advanced HTTP server configuration. | +| `max_decoding_message_size` | integer (uint) | — | 4 MiB | Maximum number of bytes to decode on each grpc stream chunk. | +| `tls` | [TlsConfig](#tlsconfig) | — | — | TLS configuration for this server. If not set, the server will not use TLS. | + +## CasServiceConfig + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `instance_name` | string | — | `""` | Used when the config references `instance_name` in the protocol. | +| `cas_store` | string | Yes | — | The store name referenced in the `stores` map in the main config. This store name referenced here may be reused multiple times. | + +## ActionCacheServiceConfig + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `instance_name` | string | — | `""` | Used when the config references `instance_name` in the protocol. | +| `ac_store` | string | Yes | — | The store name referenced in the `stores` map in the main config. This store name referenced here may be reused multiple times. | +| `read_only` | boolean | — | `false` | Whether the Action Cache store may be written to, this if set to false it is only possible to read from the Action Cache. | + +## CapabilitiesServiceConfig + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `instance_name` | string | — | `""` | Used when the config references `instance_name` in the protocol. | +| `remote_execution` | [CapabilitiesRemoteExecutionConfig](#capabilitiesremoteexecutionconfig) | — | — | Configuration for remote execution capabilities. If not set the capabilities service will inform the client that remote execution is not supported. | + +## ExecutionServiceConfig + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `instance_name` | string | — | `""` | Used when the config references `instance_name` in the protocol. | +| `cas_store` | string | Yes | — | The store name referenced in the `stores` map in the main config. This store name referenced here may be reused multiple times. This value must be a CAS store reference. | +| `scheduler` | string | Yes | — | The scheduler name referenced in the `schedulers` map in the main config. | + +## ByteStreamServiceConfig + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `instance_name` | string | — | `""` | Used when the config references `instance_name` in the protocol. | +| `cas_store` | string | Yes | — | Name of the store in the "stores" configuration. | +| `max_bytes_per_stream` | integer (uint) | — | 64KiB | Max number of bytes to send on each grpc stream chunk. According to [https://github.com/grpc/grpc.github.io/issues/371](https://github.com/grpc/grpc.github.io/issues/371) 16KiB - 64KiB is optimal. | +| `persist_stream_on_disconnect_timeout_s` | integer (uint) | — | 10 seconds | In the event a client disconnects while uploading a blob, we will hold the internal stream open for this many seconds before closing it. This allows clients that disconnect to reconnect and continue uploading the same blob. | + +## FetchServiceConfig + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `instance_name` | string | — | `""` | Used when the config references `instance_name` in the protocol. | +| `fetch_store` | string | Yes | — | The store name referenced in the `stores` map in the main config. This store name referenced here may be reused multiple times. | + +## PushServiceConfig + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `instance_name` | string | — | `""` | Used when the config references `instance_name` in the protocol. | +| `push_store` | string | Yes | — | The store name referenced in the `stores` map in the main config. This store name referenced here may be reused multiple times. | +| `read_only` | boolean | — | `false` | Whether the Action Cache store may be written to, this if set to false it is only possible to read from the Action Cache. | + +## WorkerApiConfig + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `scheduler` | string | Yes | — | The scheduler name referenced in the `schedulers` map in the main config. | + +## BepConfig + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `store` | string | Yes | — | The store to publish build events to. The store name referenced in the `stores` map in the main config. | + +## AdminConfig + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `path` | string | — | "/admin" | Path to register the admin API. If path is "/admin", and your domain is "example.com", you can reach the endpoint with: [http://example.com/admin](http://example.com/admin). | + +## HealthConfig + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `path` | string | — | "/status" | Path to register the health status check. If path is "/status", and your domain is "example.com", you can reach the endpoint with: [http://example.com/status](http://example.com/status). | +| `timeout_seconds` | integer (uint64) | — | 5s | Timeout on health checks. | + +## ErrorCode + +The possible error codes that might occur on an upstream request. + +| Value | Description | +| --- | --- | +| `"Cancelled"` | | +| `"Unknown"` | | +| `"InvalidArgument"` | | +| `"DeadlineExceeded"` | | +| `"NotFound"` | | +| `"AlreadyExists"` | | +| `"PermissionDenied"` | | +| `"ResourceExhausted"` | | +| `"FailedPrecondition"` | | +| `"Aborted"` | | +| `"OutOfRange"` | | +| `"Unimplemented"` | | +| `"Internal"` | | +| `"Unavailable"` | | +| `"DataLoss"` | | +| `"Unauthenticated"` | | + +## Lz4Config + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `block_size` | integer (uint32) | — | 65536 (64k) | Size of the blocks to compress. Higher values require more ram, but might yield slightly better compression ratios. | +| `max_decode_block_size` | integer (uint32) | — | value in `block_size` | Maximum size allowed to attempt to deserialize data into. This is needed because the `block_size` is embedded into the data so if there was a bad actor, they could upload an extremely large `block_size`'ed entry and we'd allocate a large amount of memory when retrieving the data. To prevent this from happening, we allow you to specify the maximum that we'll attempt to deserialize. | + +## ClientTlsConfig + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `ca_file` | string | — | — | Path to the certificate authority to use to validate the remote. | +| `cert_file` | string | — | — | Path to the certificate file for client authentication. | +| `key_file` | string | — | — | Path to the private key file for client authentication. | +| `use_native_roots` | boolean | — | false | If set the client will use the native roots for TLS connections. | + +## UploadCacheResultsStrategy + +| Value | Description | +| --- | --- | +| `"success_only"` | Only upload action results with an exit code of 0. | +| `"never"` | Don't upload any action results. | +| `"everything"` | Upload all action results that complete. | +| `"failures_only"` | Only upload action results that fail. | + +## ExperimentalRedisSchedulerBackend + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `redis_store` | string | Yes | — | A reference to the Redis store to use for the scheduler. Note: This MUST resolve to a `RedisSpec`. | + +## PlatformPropertyAddition + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `name` | string | Yes | — | The name of the property to add. | +| `value` | string | Yes | — | The value to assign to the property. | + +## PlatformPropertyReplacement + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `name` | string | Yes | — | The name of the property to replace. | +| `value` | string | — | — | The value to match against, if unset then any instance matches. | +| `new_name` | string | Yes | — | The new name of the property. | +| `new_value` | string | — | — | The value to assign to the property, if unset will remain the same. | + +## HttpCompressionConfig + +Note: Compressing data in the cloud rarely has a benefit, since most +cloud providers have very high bandwidth backplanes. However, for +clients not inside the data center, it might be a good idea to +compress data to and from the cloud. This will however come at a high +CPU and performance cost. If you are making remote execution share the +same CAS/AC servers as client's remote cache, you can create multiple +services with different compression settings that are served on +different ports. Then configure the non-cloud clients to use one port +and cloud-clients to use another. + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `send_compression_algorithm` | [HttpCompressionAlgorithm](#httpcompressionalgorithm) | — | `HttpCompressionAlgorithm::None` | The compression algorithm that the server will use when sending responses to clients. Enabling this will likely save a lot of data transfer, but will consume a lot of CPU and add a lot of latency. see: [https://github.com/tracemachina/nativelink/issues/109](https://github.com/tracemachina/nativelink/issues/109) | +| `accepted_compression_algorithms` | array of [HttpCompressionAlgorithm](#httpcompressionalgorithm) | Yes | {no supported compression} | The compression algorithm that the server will accept from clients. The server will broadcast the supported compression algorithms to clients and the client will choose which compression algorithm to use. Enabling this will likely save a lot of data transfer, but will consume a lot of CPU and add a lot of latency. see: [https://github.com/tracemachina/nativelink/issues/109](https://github.com/tracemachina/nativelink/issues/109) | + +## HttpServerConfig + +Advanced HTTP configuration. These generally should not be set. +For documentation on these settings, see the hyper documentation: +See: [hyper HTTP server docs](https://docs.rs/hyper/latest/hyper/server/conn/struct.Http.html) + +Note: All of these default to the default values from hyper unless otherwise +specified. + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `http2_keep_alive_interval` | integer (uint32) | — | — | Interval to send keep-alive pings via HTTP2. Note: This is in seconds. | +| `experimental_http2_max_pending_accept_reset_streams` | integer (uint32) | — | — | | +| `experimental_http2_initial_stream_window_size` | integer (uint32) | — | — | | +| `experimental_http2_initial_connection_window_size` | integer (uint32) | — | — | | +| `experimental_http2_adaptive_window` | boolean | — | — | | +| `experimental_http2_max_frame_size` | integer (uint32) | — | — | | +| `experimental_http2_max_concurrent_streams` | integer (uint32) | — | — | | +| `experimental_http2_keep_alive_timeout_s` | integer (uint32) | — | — | | +| `experimental_http2_max_send_buf_size` | integer (uint32) | — | — | | +| `experimental_http2_enable_connect_protocol` | boolean | — | — | | +| `experimental_http2_max_header_list_size` | integer (uint32) | — | — | | + +## TlsConfig + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `cert_file` | string | Yes | — | Path to the certificate file. | +| `key_file` | string | Yes | — | Path to the private key file. | +| `client_ca_file` | string | — | — | Path to the certificate authority for mTLS, if client authentication is required for this endpoint. | +| `client_crl_file` | string | — | — | Path to the certificate revocation list for mTLS, if client authentication is required for this endpoint. | + +## CapabilitiesRemoteExecutionConfig + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `scheduler` | string | Yes | — | Scheduler used to configure the capabilities of remote execution. | + +## HttpCompressionAlgorithm + +| Value | Description | +| --- | --- | +| `"none"` | No compression. | +| `"gzip"` | Zlib compression. | + +## Reading the source + +If anything here disagrees with the binary, the source wins: + +- [`stores.rs`](https://github.com/TraceMachina/nativelink/tree/v1.6.0/nativelink-config/src/stores.rs) +- [`cas_server.rs`](https://github.com/TraceMachina/nativelink/tree/v1.6.0/nativelink-config/src/cas_server.rs) +- [`schedulers.rs`](https://github.com/TraceMachina/nativelink/tree/v1.6.0/nativelink-config/src/schedulers.rs) From c9d16c3367ed1638ee7fc7426ec6ddb05746ed02 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Thu, 16 Jul 2026 12:10:16 +0100 Subject: [PATCH 04/84] docs(config-reference): regenerate for NativeLink (#2555) --- .../reference/nativelink-config/index.mdx | 866 ++-------- .../docs/reference/nativelink-config/main.mdx | 28 +- .../reference/nativelink-config/v1.0.0.mdx | 571 +------ .../reference/nativelink-config/v1.1.0.mdx | 581 +------ .../reference/nativelink-config/v1.2.0.mdx | 581 +------ .../reference/nativelink-config/v1.3.0.mdx | 581 +------ .../reference/nativelink-config/v1.3.1.mdx | 581 +------ .../reference/nativelink-config/v1.3.2.mdx | 607 +------ .../reference/nativelink-config/v1.4.0.mdx | 607 +------ .../reference/nativelink-config/v1.5.0.mdx | 607 +------ .../reference/nativelink-config/v1.5.1.mdx | 607 +------ .../reference/nativelink-config/v1.5.2.mdx | 1460 +++++++++++++++++ web/apps/docs/lib/config-versions.ts | 30 +- 13 files changed, 1779 insertions(+), 5928 deletions(-) create mode 100644 web/apps/docs/content/docs/reference/nativelink-config/v1.5.2.mdx diff --git a/web/apps/docs/content/docs/reference/nativelink-config/index.mdx b/web/apps/docs/content/docs/reference/nativelink-config/index.mdx index 62968266e..8fe00e3b5 100644 --- a/web/apps/docs/content/docs/reference/nativelink-config/index.mdx +++ b/web/apps/docs/content/docs/reference/nativelink-config/index.mdx @@ -5,14 +5,14 @@ full: true --- {/* AUTOGENERATED — do not edit by hand. - Source: nativelink-config @ v1.5.2 (6e63ef9a) + Source: nativelink-config @ v1.6.1 (23e960dc) Regenerate from web/: bun --filter @nativelink/docs gen:config-reference */} -This is the canonical NativeLink configuration reference for **v1.5.2**. +This is the canonical NativeLink configuration reference for **v1.6.1**. It is autogenerated from the Rust config crate -([`nativelink-config/src`](https://github.com/TraceMachina/nativelink/tree/v1.5.2/nativelink-config/src)) via the `build-schema` binary, so +([`nativelink-config/src`](https://github.com/TraceMachina/nativelink/tree/v1.6.1/nativelink-config/src)) via the `build-schema` binary, so it can never drift from what the binary actually deserializes. ## Top-level fields @@ -180,6 +180,55 @@ It supports the following backends: } ``` +5. **Cloudflare R2:** + R2 store uses Cloudflare's R2 service as a backend. R2 speaks the + S3 API, so this is a thin wrapper that derives the account-scoped + endpoint (`https://{account_id}.r2.cloudflarestorage.com`) for you. + + **Example JSON5 config:** + ```json5 + "experimental_cloud_object_store": { + "provider": "r2", + "account_id": "a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4", + "bucket": "nativelink-cas", + "key_prefix": "test-prefix/", + "retry": { + "max_retries": 6, + "delay": 0.3, + "jitter": 0.5 + }, + "multipart_max_concurrent_uploads": 10 + } + ``` + +6. **Oracle Cloud Infrastructure (OCI) Object Storage:** + OCI store uses Oracle Cloud Infrastructure's S3-compatible Object + Storage API. The path-style endpoint is derived from your Object + Storage `namespace` and `region` as + `https://{namespace}.compat.objectstorage.{region}.oci.customer-oci.com`. + Authenticate with a Customer Secret Key (Access Key/Secret Key pair + created under User Settings -> Customer secret keys in the OCI + console); the secret cannot be retrieved after generation, so read + it from an env var via shellexpand. + + **Example JSON5 config:** + ```json5 + "experimental_cloud_object_store": { + "provider": "oci", + "namespace": "your-object-storage-namespace", + "region": "us-phoenix-1", + "bucket": "nativelink-cas", + "access_key_id": "oci_access_key_id", + "secret_access_key": "oci_secret_access_key", + "key_prefix": "test-prefix/", + "retry": { + "max_retries": 6, + "delay": 0.3, + "jitter": 0.5 + } + } + ``` + **Type:** [ExperimentalCloudObjectSpec](#experimentalcloudobjectspec) ### `ontap_s3_existence_cache` @@ -688,7 +737,7 @@ Plus exactly one of the following variants (the key selects the variant): | Field | Type | Required | Default | Description | | --- | --- | --- | --- | --- | | `cache_type` | string | Yes | — | Low-cardinality cache type label for metrics, for example `cas` or `ac`. | -| `backend` | [StoreSpec](#storespec) | Yes | — | Store to wrap with cache operation metrics. | +| `backend` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | Store to wrap with cache operation metrics. | ## MemorySpec @@ -712,7 +761,7 @@ See [`experimental_cloud_object_store`](#experimental_cloud_object_store-1) for | Field | Type | Required | Default | Description | | --- | --- | --- | --- | --- | -| `backend` | [StoreSpec](#storespec) | Yes | — | The underlying store wrap around. All content will first flow through self before forwarding to backend. In the event there is an error detected in self, the connection to the backend will be terminated, and early termination should always cause updates to fail on the backend. | +| `backend` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | The underlying store wrap around. All content will first flow through self before forwarding to backend. In the event there is an error detected in self, the connection to the backend will be terminated, and early termination should always cause updates to fail on the backend. | | `verify_size` | boolean | — | `false` | If set the store will verify the size of the data before accepting an upload of data. | | `verify_hash` | boolean | — | `false` | If the data should be hashed and verify that the key matches the computed hash. The hash function is automatically determined based request and if not set will use the global default. | @@ -720,22 +769,22 @@ See [`experimental_cloud_object_store`](#experimental_cloud_object_store-1) for | Field | Type | Required | Default | Description | | --- | --- | --- | --- | --- | -| `backend` | [StoreSpec](#storespec) | Yes | — | The underlying store that will have it's results validated before sending to client. | -| `cas_store` | [StoreSpec](#storespec) | Yes | — | When a request is made, the results are decoded and all output digests/files are verified to exist in this CAS store before returning success. | +| `backend` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | The underlying store that will have it's results validated before sending to client. | +| `cas_store` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | When a request is made, the results are decoded and all output digests/files are verified to exist in this CAS store before returning success. | ## CompressionSpec | Field | Type | Required | Default | Description | | --- | --- | --- | --- | --- | -| `backend` | [StoreSpec](#storespec) | Yes | — | The underlying store wrap around. All content will first flow through self before forwarding to backend. In the event there is an error detected in self, the connection to the backend will be terminated, and early termination should always cause updates to fail on the backend. | +| `backend` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | The underlying store wrap around. All content will first flow through self before forwarding to backend. In the event there is an error detected in self, the connection to the backend will be terminated, and early termination should always cause updates to fail on the backend. | | `compression_algorithm` | [CompressionAlgorithm](#compressionalgorithm) | Yes | — | The compression algorithm to use. | ## DedupSpec | Field | Type | Required | Default | Description | | --- | --- | --- | --- | --- | -| `index_store` | [StoreSpec](#storespec) | Yes | — | Store used to store the index of each dedup slice. This store should generally be fast and small. | -| `content_store` | [StoreSpec](#storespec) | Yes | — | The store where the individual chunks will be uploaded. This store should generally be the slower & larger store. | +| `index_store` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | Store used to store the index of each dedup slice. This store should generally be fast and small. | +| `content_store` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | The store where the individual chunks will be uploaded. This store should generally be the slower & larger store. | | `min_size` | integer (uint32) | — | 64k | Minimum size that a chunk will be when slicing up the content. Note: This setting can be increased to improve performance because it will actually not check this number of bytes when deciding where to partition the data. | | `normal_size` | integer (uint32) | — | 256k | A best-effort attempt will be made to keep the average size of the chunks to this number. It is not a guarantee, but a slight attempt will be made. | | `max_size` | integer (uint32) | — | 512k | Maximum size a chunk is allowed to be. | @@ -745,16 +794,16 @@ See [`experimental_cloud_object_store`](#experimental_cloud_object_store-1) for | Field | Type | Required | Default | Description | | --- | --- | --- | --- | --- | -| `backend` | [StoreSpec](#storespec) | Yes | — | The underlying store wrap around. All content will first flow through self before forwarding to backend. In the event there is an error detected in self, the connection to the backend will be terminated, and early termination should always cause updates to fail on the backend. | +| `backend` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | The underlying store wrap around. All content will first flow through self before forwarding to backend. In the event there is an error detected in self, the connection to the backend will be terminated, and early termination should always cause updates to fail on the backend. | | `eviction_policy` | [EvictionPolicy](#evictionpolicy) | — | — | Policy used to evict items out of the store. Failure to set this value will cause items to never be removed from the store causing infinite memory usage. | ## FastSlowSpec | Field | Type | Required | Default | Description | | --- | --- | --- | --- | --- | -| `fast` | [StoreSpec](#storespec) | Yes | — | Fast store that will be attempted to be contacted before reaching out to the `slow` store. | +| `fast` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | Fast store that will be attempted to be contacted before reaching out to the `slow` store. | | `fast_direction` | [StoreDirection](#storedirection) | — | `"both"` | How to handle the fast store. This can be useful to set to Get for worker nodes such that results are persisted to the slow store only. | -| `slow` | [StoreSpec](#storespec) | Yes | — | If the object does not exist in the `fast` store it will try to get it from this store. | +| `slow` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | If the object does not exist in the `fast` store it will try to get it from this store. | | `slow_direction` | [StoreDirection](#storedirection) | — | `"both"` | How to handle the slow store. This can be useful if creating a diode and you wish to have an upstream read only store. | | `bypass_dedup_threshold_bytes` | integer (uint64) | — | disabled (0) | Reads of blobs at or above this size skip the leader/follower dedup map and stream straight from the slow store without populating the fast tier. `0` (the default) disables the bypass: every read goes through dedup, matching the prior behaviour. Enable it by setting a threshold — 256 MiB is a reasonable starting point for backends where large-blob dedup is a net loss (followers tend to time out anyway), but the right value is workload-dependent. | @@ -786,8 +835,8 @@ See [`experimental_cloud_object_store`](#experimental_cloud_object_store-1) for | Field | Type | Required | Default | Description | | --- | --- | --- | --- | --- | | `size` | integer (uint64) | Yes | — | Size to partition the data on. | -| `lower_store` | [StoreSpec](#storespec) | Yes | — | Store to send data when object is < (less than) size. | -| `upper_store` | [StoreSpec](#storespec) | Yes | — | Store to send data when object is >= (less than eq) size. | +| `lower_store` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | Store to send data when object is < (less than) size. | +| `upper_store` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | Store to send data when object is >= (less than eq) size. | ## GrpcSpec @@ -857,8 +906,10 @@ Configuration for `ExperimentalMongoDB` store. | --- | --- | --- | --- | --- | | `name` | string | — | {Index position in the workers list} | Name of the worker. This is give a more friendly name to a worker for logging and metric publishing. This is also the prefix of the worker id (i.e., "{name}{uuidv6}"). | | `worker_api_endpoint` | [EndpointConfig](#endpointconfig) | Yes | — | Endpoint which the worker will connect to the scheduler's `WorkerApiService`. | -| `max_action_timeout` | integer (uint) | — | 20 minutes | The maximum time an action is allowed to run. If a task requests for a timeout longer than this time limit, the task will be rejected. Value in seconds. | -| `max_upload_timeout` | integer (uint) | — | 10 minutes | Maximum time allowed for uploading action results to CAS after execution completes. If upload takes longer than this, the action fails with `DeadlineExceeded` and may be retried by the scheduler. Value in seconds. | +| `max_action_timeout_s` | integer (uint) | — | 20 minutes | The maximum time an action is allowed to run. If a task requests for a timeout longer than this time limit, the task will be rejected. Value in seconds. | +| `max_upload_timeout_s` | integer (uint) | — | 10 minutes | Maximum time allowed for uploading action results to CAS after execution completes. If upload takes longer than this, the action fails with `DeadlineExceeded` and may be retried by the scheduler. Value in seconds. | +| `max_cleanup_wait_s` | integer (uint) | — | 30 seconds | Maximum time to wait for action directory cleanup before timing out. Value in seconds. | +| `max_cleanup_backoff_ms` | integer (uint) | — | 500 milliseconds | Maximum backoff duration for exponential backoff when waiting for cleanup. Value in milliseconds. | | `max_inflight_tasks` | integer (uint64) | — | 0 (infinite tasks) | Maximum number of inflight tasks this worker can cope with. | | `timeout_handled_externally` | boolean | — | false (`NativeLink` fully handles timeouts) | If timeout is handled in `entrypoint` or another wrapper script. If set to true `NativeLink` will not honor the timeout the action requested and instead will always force kill the action after `max_action_timeout` has been reached. If this is set to false, the smaller value of the action's timeout and `max_action_timeout` will be used to which `NativeLink` will kill the action. | | `entrypoint` | string | — | {Use the command from the job request} | The command to execute on every execution request. This will be parsed as a command + arguments (not shell). Example: "run.sh" and a job with command: "sleep 5" will result in a command like: "run.sh sleep 5". | @@ -970,704 +1021,123 @@ Listener for HTTP/HTTPS/HTTP2 sockets. | `"sha256"` | Use the sha256 hash function. [https://en.wikipedia.org/wiki/SHA-2](https://en.wikipedia.org/wiki/SHA-2) | | `"blake3"` | Use the blake3 hash function. [https://en.wikipedia.org/wiki/BLAKE_(hash_function)](https://en.wikipedia.org/wiki/BLAKE_(hash_function)) | -## StoreSpec - -Plus exactly one of the following variants (the key selects the variant): +## EvictionPolicy -### `cache_metrics` +Eviction policy always works on LRU (Least Recently Used). Any time an entry +is touched it updates the timestamp. Inserts and updates will execute the +eviction policy removing any expired entries and/or the oldest entries +until the store size becomes smaller than `max_bytes`. -Cache metrics store wraps another store and emits low-cardinality -OpenTelemetry cache operation metrics for the wrapped store. +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `max_bytes` | integer (uint) | — | 0 | Maximum number of bytes before eviction takes place. Zero means never evict based on size. | +| `evict_bytes` | integer (uint) | — | 0 | When eviction starts based on hitting `max_bytes`, continue until `max_bytes - evict_bytes` is met to create a low watermark. This stops operations from thrashing when the store is close to the limit. | +| `max_seconds` | integer (uint32) | — | 0 | Maximum number of seconds for an entry to live since it was last accessed before it is evicted. Zero means never evict based on time. | +| `max_count` | integer (uint64) | — | 0 | Maximum size of the store before an eviction takes place. Zero means never evict based on count. | -This wrapper is opt-in. Stores that are not explicitly wrapped by -`cache_metrics` are constructed exactly as they are without this -wrapper and do not pay its hot-path timing or recording cost. +## Retry -**Example JSON5 config:** -```json5 -"cache_metrics": { - "cache_type": "cas", - "backend": { - "filesystem": { - "content_path": "~/.cache/nativelink/content_path-cas", - "temp_path": "~/.cache/nativelink/tmp_path-cas" - } - } +Retry configuration. This configuration is exponential and each iteration +a jitter as a percentage is applied of the calculated delay. For example: +```haskell +Retry{ + max_retries: 7, + delay: 0.1, + jitter: 0.5, } ``` +will result in: -**Type:** [CacheMetricsSpec](#cachemetricsspec) - -### `memory` - -Memory store will store all data in a hash map in memory. +| Attempt | Delay | +| --- | --- | +| 1 | 0 ms | +| 2 | 75 to 125 ms | +| 3 | 150 to 250 ms | +| 4 | 300 to 500 ms | +| 5 | 600 ms to 1 s | +| 6 | 1.2 to 2 s | +| 7 | 2.4 to 4 s | +| 8 | 4.8 to 8 s | -**Example JSON5 config:** -```json5 -"memory": { - "eviction_policy": { - "max_bytes": "10mb", - } -} -``` +The total delay is additive, so this example produces 9.525 to 15.875 s of total delay for a single request. -**Type:** [MemorySpec](#memoryspec) +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `max_retries` | integer (uint) | — | `0` | Maximum number of retries until retrying stops. Setting this to zero will always attempt 1 time, but not retry. | +| `delay` | number (float) | — | `0` | Delay in seconds for exponential back off. | +| `jitter` | number (float) | — | `0` | Amount of jitter to add as a percentage in decimal form. This will change the formula like: | +| `retry_on_errors` | array of [ErrorCode](#errorcode) | — | — | A list of error codes to retry on, if this is not set then the default error codes to retry on are used. These default codes are the most likely to be non-permanent: `Unknown`; `Cancelled`; `DeadlineExceeded`; `ResourceExhausted`; `Aborted`; `Internal`; `Unavailable`; `DataLoss` | -### `experimental_cloud_object_store` +## ExperimentalOntapS3Spec -A generic blob store that will store files on the cloud -provider. This configuration will never delete files, so you are -responsible for purging old files in other ways. -It supports the following backends: +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `endpoint` | string | Yes | — | | +| `vserver_name` | string | Yes | — | | +| `bucket` | string | Yes | — | | +| `root_certificates` | string | — | — | | +| `key_prefix` | string | — | — | If you wish to prefix the location in the bucket. If None, no prefix will be used. | +| `retry` | [Retry](#retry) | — | — | Retry configuration to use when a network request fails. | +| `consider_expired_after_s` | integer (uint32) | — | 0 | If the number of seconds since the `last_modified` time of the object is greater than this value, the object will not be considered "existing". This allows for external tools to delete objects that have not been uploaded in a long time. If a client receives a `NotFound` the client should re-upload the object. | +| `max_retry_buffer_per_request` | integer (uint) | — | 5MB | The maximum buffer size to retain in case of a retryable error during upload. Setting this to zero will disable upload buffering; this means that in the event of a failure during upload, the entire upload will be aborted and the client will likely receive an error. | +| `multipart_max_concurrent_uploads` | integer (uint) | — | 10 | Maximum number of concurrent `UploadPart` requests per `MultipartUpload`. | +| `insecure_allow_http` | boolean | — | false | Allow unencrypted HTTP connections. Only use this for local testing. | +| `disable_http2` | boolean | — | false | Disable HTTP/2 connections and only use HTTP/1.1. Default client configuration will have HTTP/1.1 and HTTP/2 enabled for connection schemes. HTTP/2 should be disabled if environments have poor support or performance related to HTTP/2. Safe to keep default unless underlying network environment, S3, or GCS API servers specify otherwise. | -1. **Amazon S3:** - S3 store will use Amazon's S3 service as a backend to store - the files. This configuration can be used to share files - across multiple instances. Uses system certificates for TLS - verification via `rustls-platform-verifier`. +## CompressionAlgorithm - **Example JSON5 config:** - ```json5 - "experimental_cloud_object_store": { - "provider": "aws", - "region": "eu-north-1", - "bucket": "crossplane-bucket-af79aeca9", - "key_prefix": "test-prefix-index/", - "retry": { - "max_retries": 6, - "delay": 0.3, - "jitter": 0.5 - }, - "multipart_max_concurrent_uploads": 10 - } - ``` +Plus exactly one of the following variants (the key selects the variant): -2. **Google Cloud Storage:** - GCS store uses Google's GCS service as a backend to store - the files. This configuration can be used to share files - across multiple instances. +### `lz4` - **Example JSON5 config:** - ```json5 - "experimental_cloud_object_store": { - "provider": "gcs", - "bucket": "test-bucket", - "key_prefix": "test-prefix-index/", - "retry": { - "max_retries": 6, - "delay": 0.3, - "jitter": 0.5 - }, - "multipart_max_concurrent_uploads": 10 - } - ``` +LZ4 compression algorithm is extremely fast for compression and +decompression, however does not perform very well in compression +ratio. In most cases build artifacts are highly compressible, however +lz4 is quite good at aborting early if the data is not deemed very +compressible. -3. **Azure Blob Store:** - Azure Blob store will use Microsoft's Azure Blob service as a - backend to store the files. This configuration can be used to - share files across multiple instances. +see: [https://lz4.github.io/lz4/](https://lz4.github.io/lz4/) - **Example JSON5 config:** - ```json5 - "experimental_cloud_object_store": { - "provider": "azure", - "account_name": "cloudshell1393657559", - "container": "simple-test-container", - "key_prefix": "folder/", - "retry": { - "max_retries": 6, - "delay": 0.3, - "jitter": 0.5 - }, - "multipart_max_concurrent_uploads": 10 - } - ``` +**Type:** [Lz4Config](#lz4config) -4. **`NetApp` ONTAP S3** - `NetApp` ONTAP S3 store will use ONTAP's S3-compatible storage as a backend - to store files. This store is specifically configured for ONTAP's S3 requirements - including custom TLS configuration, credentials management, and proper vserver - configuration. +## StoreDirection - This store uses AWS environment variables for credentials: - - `AWS_ACCESS_KEY_ID` - - `AWS_SECRET_ACCESS_KEY` - - `AWS_DEFAULT_REGION` +| Value | Description | +| --- | --- | +| `"both"` | The store operates normally and all get and put operations are handled by it. | +| `"update"` | Update operations will cause persistence to this store, but Get operations will be ignored. This only makes sense on the fast store as the slow store will never get written to on Get anyway. | +| `"get"` | Get operations will cause persistence to this store, but Update operations will be ignored. | +| `"read_only"` | Operate as a read only store, only really makes sense if there's another way to write to it. | - **Example JSON5 config:** - ```json5 - "experimental_cloud_object_store": { - "provider": "ontap", - "endpoint": "https://ontap-s3-endpoint:443", - "vserver_name": "your-vserver", - "bucket": "your-bucket", - "root_certificates": "/path/to/certs.pem", // Optional - "key_prefix": "test-prefix/", // Optional - "retry": { - "max_retries": 6, - "delay": 0.3, - "jitter": 0.5 - }, - "multipart_max_concurrent_uploads": 10 - } - ``` +## ShardConfig -**Type:** [ExperimentalCloudObjectSpec](#experimentalcloudobjectspec) +Configuration for an individual shard of the store. -### `ontap_s3_existence_cache` +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `store` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | Store to shard the data to. | +| `weight` | integer (uint32) | — | 1 | The weight of the store. This is used to determine how much data should be sent to the store. The actual percentage is the sum of all the store's weights divided by the individual store's weight. | -ONTAP S3 Existence Cache provides a caching layer on top of the ONTAP S3 store -to optimize repeated existence checks. It maintains an in-memory cache of object -digests and periodically syncs this cache to disk for persistence. +## GrpcEndpoint -The cache helps reduce latency for repeated calls to check object existence, -while still ensuring eventual consistency with the underlying ONTAP S3 store. +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `address` | string | Yes | — | The endpoint address (i.e. grpc://example.com:443 or grpcs://example.com:443). | +| `tls_config` | [ClientTlsConfig](#clienttlsconfig) | — | — | The TLS configuration to use to connect to the endpoint (if grpcs). | +| `concurrency_limit` | integer (uint) | — | — | The maximum concurrency to allow on this endpoint. | +| `connect_timeout_s` | integer (uint64) | — | 30 seconds | Timeout for establishing a TCP connection to the endpoint (seconds). | +| `tcp_keepalive_s` | integer (uint64) | — | 30 seconds | TCP keepalive interval (seconds). Sends TCP keepalive probes at this interval to detect dead connections at the OS level. | +| `http2_keepalive_interval_s` | integer (uint64) | — | 30 seconds | HTTP/2 keepalive interval (seconds). Sends HTTP/2 PING frames at this interval to detect dead connections at the application level. | +| `http2_keepalive_timeout_s` | integer (uint64) | — | 20 seconds | HTTP/2 keepalive timeout (seconds). If a PING response is not received within this duration, the connection is considered dead. | -Example JSON5 config: -```json5 -"ontap_s3_existence_cache": { - "index_path": "/path/to/cache/index.json", - "sync_interval_seconds": 300, - "backend": { - "endpoint": "https://ontap-s3-endpoint:443", - "vserver_name": "your-vserver", - "bucket": "your-bucket", - "key_prefix": "test-prefix/" - } -} -``` +## StoreType -**Type:** [OntapS3ExistenceCacheSpec](#ontaps3existencecachespec) +| Value | Description | +| --- | --- | +| `"cas"` | The store is content addressable storage. | +| `"ac"` | The store is an action cache. | -### `verify` - -Verify store is used to apply verifications to an underlying -store implementation. It is strongly encouraged to validate -as much data as you can before accepting data from a client, -failing to do so may cause the data in the store to be -populated with invalid data causing all kinds of problems. - -The suggested configuration is to have the CAS validate the -hash and size and the AC validate nothing. - -**Example JSON5 config:** -```json5 -"verify": { - "backend": { - "memory": { - "eviction_policy": { - "max_bytes": "500mb" - } - }, - }, - "verify_size": true, - "verify_hash": true -} -``` - -**Type:** [VerifySpec](#verifyspec) - -### `completeness_checking` - -Completeness checking store verifies if the -output files & folders exist in the CAS before forwarding -the request to the underlying store. -Note: This store should only be used on AC stores. - -**Example JSON5 config:** -```json5 -"completeness_checking": { - "backend": { - "filesystem": { - "content_path": "~/.cache/nativelink/content_path-ac", - "temp_path": "~/.cache/nativelink/tmp_path-ac", - "eviction_policy": { - "max_bytes": "500mb", - } - } - }, - "cas_store": { - "ref_store": { - "name": "CAS_MAIN_STORE" - } - } -} -``` - -**Type:** [CompletenessCheckingSpec](#completenesscheckingspec) - -### `compression` - -A compression store that will compress the data inbound and -outbound. There will be a non-trivial cost to compress and -decompress the data, but in many cases if the final store is -a store that requires network transport and/or storage space -is a concern it is often faster and more efficient to use this -store before those stores. - -**Example JSON5 config:** -```json5 -"compression": { - "compression_algorithm": { - "lz4": {} - }, - "backend": { - "filesystem": { - "content_path": "/tmp/nativelink/data/content_path-cas", - "temp_path": "/tmp/nativelink/data/tmp_path-cas", - "eviction_policy": { - "max_bytes": "2gb", - } - } - } -} -``` - -**Type:** [CompressionSpec](#compressionspec) - -### `dedup` - -A dedup store will take the inputs and run a rolling hash -algorithm on them to slice the input into smaller parts then -run a sha256 algorithm on the slice and if the object doesn't -already exist, upload the slice to the `content_store` using -a new digest of just the slice. Once all parts exist, an -Action-Cache-like digest will be built and uploaded to the -`index_store` which will contain a reference to each -chunk/digest of the uploaded file. Downloading a request will -first grab the index from the `index_store`, and forward the -download content of each chunk as if it were one file. - -This store is exceptionally good when the following conditions -are met: -* Content is mostly the same (inserts, updates, deletes are ok) -* Content is not compressed or encrypted -* Uploading or downloading from `content_store` is the bottleneck. - -Note: This store pairs well when used with `CompressionSpec` as -the `content_store`, but never put `DedupSpec` as the backend of -`CompressionSpec` as it will negate all the gains. - -Note: When running `.has()` on this store, it will only check -to see if the entry exists in the `index_store` and not check -if the individual chunks exist in the `content_store`. - -**Example JSON5 config:** -```json5 -"dedup": { - "index_store": { - "memory": { - "eviction_policy": { - "max_bytes": "1GB", - } - } - }, - "content_store": { - "compression": { - "compression_algorithm": { - "lz4": {} - }, - "backend": { - "fast_slow": { - "fast": { - "memory": { - "eviction_policy": { - "max_bytes": "500MB", - } - } - }, - "slow": { - "filesystem": { - "content_path": "/tmp/nativelink/data/content_path-content", - "temp_path": "/tmp/nativelink/data/tmp_path-content", - "eviction_policy": { - "max_bytes": "2gb" - } - } - } - } - } - } - } -} -``` - -**Type:** [DedupSpec](#dedupspec) - -### `existence_cache` - -Existence store will wrap around another store and cache calls -to has so that subsequent `has_with_results` calls will be -faster. This is useful for cases when you have a store that -is slow to respond to has calls. -Note: This store should only be used on CAS stores. - -**Example JSON5 config:** -```json5 -"existence_cache": { - "backend": { - "memory": { - "eviction_policy": { - "max_bytes": "500mb", - } - } - }, - // Note this is the existence store policy, not the backend policy - "eviction_policy": { - "max_seconds": 100, - } -} -``` - -**Type:** [ExistenceCacheSpec](#existencecachespec) - -### `fast_slow` - -`FastSlow` store will first try to fetch the data from the `fast` -store and then if it does not exist try the `slow` store. -When the object does exist in the `slow` store, it will copy -the data to the `fast` store while returning the data. -This store should be thought of as a store that "buffers" -the data to the `fast` store. -On uploads it will mirror data to both `fast` and `slow` stores. - -WARNING: If you need data to always exist in the `slow` store -for something like remote execution, be careful because this -store will never check to see if the objects exist in the -`slow` store if it exists in the `fast` store (i.e., it assumes -that if an object exists in the `fast` store it will exist in -the `slow` store). - -***Example JSON5 config:*** -```json5 -"fast_slow": { - "fast": { - "filesystem": { - "content_path": "/tmp/nativelink/data/content_path-index", - "temp_path": "/tmp/nativelink/data/tmp_path-index", - "eviction_policy": { - "max_bytes": "500mb", - } - } - }, - "slow": { - "filesystem": { - "content_path": "/tmp/nativelink/data/content_path-index", - "temp_path": "/tmp/nativelink/data/tmp_path-index", - "eviction_policy": { - "max_bytes": "500mb", - } - } - } -} -``` - -**Type:** [FastSlowSpec](#fastslowspec) - -### `shard` - -Shards the data to multiple stores. This is useful for cases -when you want to distribute the load across multiple stores. -The digest hash is used to determine which store to send the -data to. - -**Example JSON5 config:** -```json5 -"shard": { - "stores": [ - { - "store": { - "memory": { - "eviction_policy": { - "max_bytes": "10mb" - }, - }, - }, - "weight": 1 - }] -} -``` - -**Type:** [ShardSpec](#shardspec) - -### `filesystem` - -Stores the data on the filesystem. This store is designed for -local persistent storage. Restarts of this program should restore -the previous state, meaning anything uploaded will be persistent -as long as the filesystem integrity holds. - -**Example JSON5 config:** -```json5 -"filesystem": { - "content_path": "/tmp/nativelink/data-worker-test/content_path-cas", - "temp_path": "/tmp/nativelink/data-worker-test/tmp_path-cas", - "eviction_policy": { - "max_bytes": "10gb", - } -} -``` - -**Type:** [FilesystemSpec](#filesystemspec) - -### `ref_store` - -Store used to reference a store in the root store manager. -This is useful for cases when you want to share a store in different -nested stores. Example, you may want to share the same memory store -used for the action cache, but use a `FastSlowSpec` and have the fast -store also share the memory store for efficiency. - -**Example JSON5 config:** -```json5 -"ref_store": { - "name": "FS_CONTENT_STORE" -} -``` - -**Type:** [RefSpec](#refspec) - -### `size_partitioning` - -Uses the size field of the digest to separate which store to send the -data. This is useful for cases when you'd like to put small objects -in one store and large objects in another store. This should only be -used if the size field is the real size of the content, in other -words, don't use on AC (Action Cache) stores. Any store where you can -safely use `VerifySpec.verify_size = true`, this store should be safe -to use (i.e., CAS stores). - -**Example JSON5 config:** -```json5 -"size_partitioning": { - "size": "128mib", - "lower_store": { - "memory": { - "eviction_policy": { - "max_bytes": "${NATIVELINK_CAS_MEMORY_CONTENT_LIMIT:-100mb}" - } - } - }, - "upper_store": { - /// This store discards data larger than 128mib. - "noop": {} - } -} -``` - -**Type:** [SizePartitioningSpec](#sizepartitioningspec) - -### `grpc` - -This store will pass-through calls to another GRPC store. This store -is not designed to be used as a sub-store of another store, but it -does satisfy the interface and will likely work. - -One major GOTCHA is that some stores use a special function on this -store to get the size of the underlying object, which is only reliable -when this store is serving the a CAS store, not an AC store. If using -this store directly without being a child of any store there are no -side effects and is the most efficient way to use it. - -**Example JSON5 config:** -```json5 -"grpc": { - "instance_name": "main", - "endpoints": [ - {"address": "grpc://${CAS_ENDPOINT:-127.0.0.1}:50051"} - ], - "connections_per_endpoint": "5", - "rpc_timeout_s": "5m", - "store_type": "ac", - // Static headers attached to every outgoing request to the upstream - // remote cache. Useful for fixed service-account credentials. - "headers": { - "authorization": "Bearer my-static-token" - }, - // Header names to copy from the inbound client request and forward to - // the upstream remote cache. Use this to pass through dynamic - // credentials such as a JWT sent by the build client. - "forward_headers": ["authorization", "x-custom-token"] -} -``` - -**Type:** [GrpcSpec](#grpcspec) - -### `redis_store` - -Stores data in any stores compatible with Redis APIs. - -Pairs well with `SizePartitioning` and/or `FastSlow` stores. -Ideal for accepting small object sizes as most Redis store -services have a max file upload of between 256Mb-512Mb. - -**Example JSON5 config:** -```json5 -"redis_store": { - "addresses": [ - "redis://127.0.0.1:6379/", - ], - "max_client_permits": 1000, -} -``` - -**Type:** [RedisSpec](#redisspec) - -### `noop` - -Noop store is a store that sends streams into the void and all data -retrieval will return 404 (`NotFound`). This can be useful for cases -where you may need to partition your data and part of your data needs -to be discarded. - -**Example JSON5 config:** -```json5 -"noop": {} -``` - -**Type:** [NoopSpec](#noopspec) - -### `experimental_mongo` - -Experimental `MongoDB` store implementation. - -This store uses `MongoDB` as a backend for storing data. It supports -both CAS (Content Addressable Storage) and scheduler data with -optional change streams for real-time updates. - -**Example JSON5 config:** -```json5 -"experimental_mongo": { - "connection_string": "mongodb://localhost:27017", - "database": "nativelink", - "cas_collection": "cas", - "key_prefix": "cas:", - "read_chunk_size": 65536, - "max_concurrent_uploads": 10, - "enable_change_streams": false, - "max_requests": "100" -} -``` - -**Type:** [ExperimentalMongoSpec](#experimentalmongospec) - -## EvictionPolicy - -Eviction policy always works on LRU (Least Recently Used). Any time an entry -is touched it updates the timestamp. Inserts and updates will execute the -eviction policy removing any expired entries and/or the oldest entries -until the store size becomes smaller than `max_bytes`. - -| Field | Type | Required | Default | Description | -| --- | --- | --- | --- | --- | -| `max_bytes` | integer (uint) | — | 0 | Maximum number of bytes before eviction takes place. Zero means never evict based on size. | -| `evict_bytes` | integer (uint) | — | 0 | When eviction starts based on hitting `max_bytes`, continue until `max_bytes - evict_bytes` is met to create a low watermark. This stops operations from thrashing when the store is close to the limit. | -| `max_seconds` | integer (uint32) | — | 0 | Maximum number of seconds for an entry to live since it was last accessed before it is evicted. Zero means never evict based on time. | -| `max_count` | integer (uint64) | — | 0 | Maximum size of the store before an eviction takes place. Zero means never evict based on count. | - -## Retry - -Retry configuration. This configuration is exponential and each iteration -a jitter as a percentage is applied of the calculated delay. For example: -```haskell -Retry{ - max_retries: 7, - delay: 0.1, - jitter: 0.5, -} -``` -will result in: - -| Attempt | Delay | -| --- | --- | -| 1 | 0 ms | -| 2 | 75 to 125 ms | -| 3 | 150 to 250 ms | -| 4 | 300 to 500 ms | -| 5 | 600 ms to 1 s | -| 6 | 1.2 to 2 s | -| 7 | 2.4 to 4 s | -| 8 | 4.8 to 8 s | - -The total delay is additive, so this example produces 9.525 to 15.875 s of total delay for a single request. - -| Field | Type | Required | Default | Description | -| --- | --- | --- | --- | --- | -| `max_retries` | integer (uint) | — | `0` | Maximum number of retries until retrying stops. Setting this to zero will always attempt 1 time, but not retry. | -| `delay` | number (float) | — | `0` | Delay in seconds for exponential back off. | -| `jitter` | number (float) | — | `0` | Amount of jitter to add as a percentage in decimal form. This will change the formula like: | -| `retry_on_errors` | array of [ErrorCode](#errorcode) | — | — | A list of error codes to retry on, if this is not set then the default error codes to retry on are used. These default codes are the most likely to be non-permanent: `Unknown`; `Cancelled`; `DeadlineExceeded`; `ResourceExhausted`; `Aborted`; `Internal`; `Unavailable`; `DataLoss` | - -## ExperimentalOntapS3Spec - -| Field | Type | Required | Default | Description | -| --- | --- | --- | --- | --- | -| `endpoint` | string | Yes | — | | -| `vserver_name` | string | Yes | — | | -| `bucket` | string | Yes | — | | -| `root_certificates` | string | — | — | | -| `key_prefix` | string | — | — | If you wish to prefix the location in the bucket. If None, no prefix will be used. | -| `retry` | [Retry](#retry) | — | — | Retry configuration to use when a network request fails. | -| `consider_expired_after_s` | integer (uint32) | — | 0 | If the number of seconds since the `last_modified` time of the object is greater than this value, the object will not be considered "existing". This allows for external tools to delete objects that have not been uploaded in a long time. If a client receives a `NotFound` the client should re-upload the object. | -| `max_retry_buffer_per_request` | integer (uint) | — | 5MB | The maximum buffer size to retain in case of a retryable error during upload. Setting this to zero will disable upload buffering; this means that in the event of a failure during upload, the entire upload will be aborted and the client will likely receive an error. | -| `multipart_max_concurrent_uploads` | integer (uint) | — | 10 | Maximum number of concurrent `UploadPart` requests per `MultipartUpload`. | -| `insecure_allow_http` | boolean | — | false | Allow unencrypted HTTP connections. Only use this for local testing. | -| `disable_http2` | boolean | — | false | Disable HTTP/2 connections and only use HTTP/1.1. Default client configuration will have HTTP/1.1 and HTTP/2 enabled for connection schemes. HTTP/2 should be disabled if environments have poor support or performance related to HTTP/2. Safe to keep default unless underlying network environment, S3, or GCS API servers specify otherwise. | - -## CompressionAlgorithm - -Plus exactly one of the following variants (the key selects the variant): - -### `lz4` - -LZ4 compression algorithm is extremely fast for compression and -decompression, however does not perform very well in compression -ratio. In most cases build artifacts are highly compressible, however -lz4 is quite good at aborting early if the data is not deemed very -compressible. - -see: [https://lz4.github.io/lz4/](https://lz4.github.io/lz4/) - -**Type:** [Lz4Config](#lz4config) - -## StoreDirection - -| Value | Description | -| --- | --- | -| `"both"` | The store operates normally and all get and put operations are handled by it. | -| `"update"` | Update operations will cause persistence to this store, but Get operations will be ignored. This only makes sense on the fast store as the slow store will never get written to on Get anyway. | -| `"get"` | Get operations will cause persistence to this store, but Update operations will be ignored. | -| `"read_only"` | Operate as a read only store, only really makes sense if there's another way to write to it. | - -## ShardConfig - -Configuration for an individual shard of the store. - -| Field | Type | Required | Default | Description | -| --- | --- | --- | --- | --- | -| `store` | [StoreSpec](#storespec) | Yes | — | Store to shard the data to. | -| `weight` | integer (uint32) | — | 1 | The weight of the store. This is used to determine how much data should be sent to the store. The actual percentage is the sum of all the store's weights divided by the individual store's weight. | - -## GrpcEndpoint - -| Field | Type | Required | Default | Description | -| --- | --- | --- | --- | --- | -| `address` | string | Yes | — | The endpoint address (i.e. grpc://example.com:443 or grpcs://example.com:443). | -| `tls_config` | [ClientTlsConfig](#clienttlsconfig) | — | — | The TLS configuration to use to connect to the endpoint (if grpcs). | -| `concurrency_limit` | integer (uint) | — | — | The maximum concurrency to allow on this endpoint. | -| `connect_timeout_s` | integer (uint64) | — | 30 seconds | Timeout for establishing a TCP connection to the endpoint (seconds). | -| `tcp_keepalive_s` | integer (uint64) | — | 30 seconds | TCP keepalive interval (seconds). Sends TCP keepalive probes at this interval to detect dead connections at the OS level. | -| `http2_keepalive_interval_s` | integer (uint64) | — | 30 seconds | HTTP/2 keepalive interval (seconds). Sends HTTP/2 PING frames at this interval to detect dead connections at the application level. | -| `http2_keepalive_timeout_s` | integer (uint64) | — | 20 seconds | HTTP/2 keepalive timeout (seconds). If a PING response is not received within this duration, the connection is considered dead. | - -## StoreType - -| Value | Description | -| --- | --- | -| `"cas"` | The store is content addressable storage. | -| `"ac"` | The store is an action cache. | - -## RedisMode +## RedisMode | Value | Description | | --- | --- | @@ -1827,6 +1297,7 @@ If a property is found, then replace it with another one. | --- | --- | --- | --- | --- | | `instance_name` | string | — | `""` | Used when the config references `instance_name` in the protocol. | | `cas_store` | string | Yes | — | The store name referenced in the `stores` map in the main config. This store name referenced here may be reused multiple times. | +| `experimental_chunking` | [CasChunkingConfig](#caschunkingconfig) | — | not set — chunking RPCs are rejected, nothing is advertised, | Optional and experimental: enables the REAPI `SplitBlob`/`SpliceBlob` RPCs used by content-defined chunking clients (e.g. Bazel's `--experimental_remote_cache_chunking`). When set, the capabilities service advertises blob split/splice support and `FastCDC` 2020 parameters for this instance. When `cas_store` is a grpc store the RPCs are forwarded to the backend (which must support chunking with matching parameters); otherwise they are served locally. | ## ActionCacheServiceConfig @@ -1842,6 +1313,7 @@ If a property is found, then replace it with another one. | --- | --- | --- | --- | --- | | `instance_name` | string | — | `""` | Used when the config references `instance_name` in the protocol. | | `remote_execution` | [CapabilitiesRemoteExecutionConfig](#capabilitiesremoteexecutionconfig) | — | — | Configuration for remote execution capabilities. If not set the capabilities service will inform the client that remote execution is not supported. | +| `remote_cache_compression` | boolean | — | — | Whether this instance supports Bazel remote cache compression. When enabled, the capabilities service advertises zstd wire compression and the ByteStream/CAS services accept REAPI compressed-blobs/zstd data. | ## ExecutionServiceConfig @@ -1858,7 +1330,7 @@ If a property is found, then replace it with another one. | `instance_name` | string | — | `""` | Used when the config references `instance_name` in the protocol. | | `cas_store` | string | Yes | — | Name of the store in the "stores" configuration. | | `max_bytes_per_stream` | integer (uint) | — | 64KiB | Max number of bytes to send on each grpc stream chunk. According to [https://github.com/grpc/grpc.github.io/issues/371](https://github.com/grpc/grpc.github.io/issues/371) 16KiB - 64KiB is optimal. | -| `persist_stream_on_disconnect_timeout` | integer (uint) | — | 10 seconds | In the event a client disconnects while uploading a blob, we will hold the internal stream open for this many seconds before closing it. This allows clients that disconnect to reconnect and continue uploading the same blob. | +| `persist_stream_on_disconnect_timeout_s` | integer (uint) | — | 10 seconds | In the event a client disconnects while uploading a blob, we will hold the internal stream open for this many seconds before closing it. This allows clients that disconnect to reconnect and continue uploading the same blob. | ## FetchServiceConfig @@ -2005,7 +1477,7 @@ specified. | `experimental_http2_adaptive_window` | boolean | — | — | | | `experimental_http2_max_frame_size` | integer (uint32) | — | — | | | `experimental_http2_max_concurrent_streams` | integer (uint32) | — | — | | -| `experimental_http2_keep_alive_timeout` | integer (uint32) | — | — | Note: This is in seconds. | +| `experimental_http2_keep_alive_timeout_s` | integer (uint32) | — | — | | | `experimental_http2_max_send_buf_size` | integer (uint32) | — | — | | | `experimental_http2_enable_connect_protocol` | boolean | — | — | | | `experimental_http2_max_header_list_size` | integer (uint32) | — | — | | @@ -2019,6 +1491,14 @@ specified. | `client_ca_file` | string | — | — | Path to the certificate authority for mTLS, if client authentication is required for this endpoint. | | `client_crl_file` | string | — | — | Path to the certificate revocation list for mTLS, if client authentication is required for this endpoint. | +## CasChunkingConfig + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `index_store` | string | — | — | The store name referenced in the `stores` map in the main config used to persist blob-to-chunks layouts. Keys are the digests of the original blobs and values are serialized chunk layouts (which do not hash to those digests), so this store MUST NOT perform content digest verification and MUST NOT be the same store as `cas_store` — writing layouts into the CAS would overwrite blob content. Using the same store name as `cas_store` is rejected at startup. | +| `avg_chunk_size_bytes` | integer (uint64) | — | 524288 (512 KiB) | The average chunk size in bytes advertised to clients through the `FastCDC` 2020 capability parameters and used for server-side chunking in `SplitBlob`. Clients derive the minimum and maximum chunk sizes from this value (avg / 4 and avg * 4). The value must be between 1 KiB and 1 MiB. | +| `max_chunk_count` | integer (uint64) | — | 50000 | Maximum number of chunks accepted in a `SpliceBlob` request or produced by on-demand chunking in `SplitBlob`. Blobs that would produce more chunks are served without chunking (`SplitBlob` returns `NOT_FOUND` and clients fall back to a regular download). This bounds the size of stored chunk layouts and of `SplitBlobResponse` messages (roughly 80-140 bytes per chunk). At the default average chunk size the default cap supports blobs up to ~25 GiB; note that values above ~50000 may produce responses that exceed default gRPC message size limits on clients. | + ## CapabilitiesRemoteExecutionConfig | Field | Type | Required | Default | Description | @@ -2036,6 +1516,6 @@ specified. If anything here disagrees with the binary, the source wins: -- [`stores.rs`](https://github.com/TraceMachina/nativelink/tree/v1.5.2/nativelink-config/src/stores.rs) -- [`cas_server.rs`](https://github.com/TraceMachina/nativelink/tree/v1.5.2/nativelink-config/src/cas_server.rs) -- [`schedulers.rs`](https://github.com/TraceMachina/nativelink/tree/v1.5.2/nativelink-config/src/schedulers.rs) +- [`stores.rs`](https://github.com/TraceMachina/nativelink/tree/v1.6.1/nativelink-config/src/stores.rs) +- [`cas_server.rs`](https://github.com/TraceMachina/nativelink/tree/v1.6.1/nativelink-config/src/cas_server.rs) +- [`schedulers.rs`](https://github.com/TraceMachina/nativelink/tree/v1.6.1/nativelink-config/src/schedulers.rs) diff --git a/web/apps/docs/content/docs/reference/nativelink-config/main.mdx b/web/apps/docs/content/docs/reference/nativelink-config/main.mdx index 3ceba63c6..adfbe5b45 100644 --- a/web/apps/docs/content/docs/reference/nativelink-config/main.mdx +++ b/web/apps/docs/content/docs/reference/nativelink-config/main.mdx @@ -5,7 +5,7 @@ full: true --- {/* AUTOGENERATED — do not edit by hand. - Source: nativelink-config @ main (1a28c5fe) + Source: nativelink-config @ main (f871377c) Regenerate from web/: bun --filter @nativelink/docs gen:config-reference */} @@ -451,7 +451,7 @@ On uploads it will mirror data to both `fast` and `slow` stores. WARNING: If you need data to always exist in the `slow` store for something like remote execution, be careful because this store will never check to see if the objects exist in the -`slow` store if it exists in the `fast` store (i.e., it assumes +`slow` store if it exists in the `fast` store (i.e. it assumes that if an object exists in the `fast` store it will exist in the `slow` store). @@ -552,7 +552,7 @@ in one store and large objects in another store. This should only be used if the size field is the real size of the content, in other words, don't use on AC (Action Cache) stores. Any store where you can safely use `VerifySpec.verify_size = true`, this store should be safe -to use (i.e., CAS stores). +to use (i.e. CAS stores). **Example JSON5 config:** ```json5 @@ -576,7 +576,7 @@ to use (i.e., CAS stores). ### `grpc` -This store will pass-through calls to another GRPC store. This store +This store will pass-through calls to another gRPC store. This store is not designed to be used as a sub-store of another store, but it does satisfy the interface and will likely work. @@ -817,8 +817,8 @@ See [`experimental_cloud_object_store`](#experimental_cloud_object_store-1) for | Field | Type | Required | Default | Description | | --- | --- | --- | --- | --- | -| `content_path` | string | Yes | — | Path on the system where to store the actual content. This is where the bulk of the data will be placed. On service startup this folder will be scanned and all files will be added to the cache. In the event one of the files doesn't match the criteria, the file will be deleted. | -| `temp_path` | string | Yes | — | A temporary location of where files that are being uploaded or deleted will be placed while the content cannot be guaranteed to be accurate. This location must be on the same block device as `content_path` so atomic moves can happen (i.e., move without copy). All files in this folder will be deleted on every startup. | +| `content_path` | string | Yes | — | Path on the system where to store the actual content. This is where the bulk of the data will be placed. On service boot this folder will be scanned and all files will be added to the cache. In the event one of the files doesn't match the criteria, the file will be deleted. | +| `temp_path` | string | Yes | — | A temporary location of where files that are being uploaded or deleted will be placed while the content cannot be guaranteed to be accurate. This location must be on the same block device as `content_path` so atomic moves can happen (i.e. move without copy). All files in this folder will be deleted on every startup. | | `read_buffer_size` | integer (uint32) | — | 32k | Buffer size to use when reading files. Generally this should be left to the default value except for testing. | | `eviction_policy` | [EvictionPolicy](#evictionpolicy) | — | — | Policy used to evict items out of the store. Failure to set this value will cause items to never be removed from the store causing infinite memory usage. | | `block_size` | integer (uint64) | — | 4kb | The block size of the filesystem for the running machine value is used to determine an entry's actual size on disk consumed For a 4KB block size filesystem, a 1B file actually consumes 4KB | @@ -842,12 +842,12 @@ See [`experimental_cloud_object_store`](#experimental_cloud_object_store-1) for | Field | Type | Required | Default | Description | | --- | --- | --- | --- | --- | -| `instance_name` | string | — | `""` | Instance name for GRPC calls. Proxy calls will have the `instance_name` changed to this. | +| `instance_name` | string | — | `""` | Instance name for gRPC calls. Proxy calls will have the `instance_name` changed to this. | | `endpoints` | array of [GrpcEndpoint](#grpcendpoint) | Yes | — | The endpoint of the grpc connection. | | `store_type` | [StoreType](#storetype) | Yes | — | The type of the upstream store, this ensures that the correct server calls are made. | | `retry` | [Retry](#retry) | — | — | Retry configuration to use when a network request fails. | | `max_concurrent_requests` | integer (uint) | — | `0` | Limit the number of simultaneous upstream requests to this many. A value of zero is treated as unlimited. If the limit is reached the request is queued. | -| `connections_per_endpoint` | integer (uint) | — | `0` | The number of connections to make to each specified endpoint to balance the load over multiple TCP connections. Default 1. | +| `connections_per_endpoint` | integer (uint) | — | 1 | The number of connections to make to each specified endpoint to balance the load over multiple TCP connections. | | `rpc_timeout_s` | integer (uint64) | — | 0 (disabled) | Maximum time (seconds) allowed for a single RPC request (e.g. a `ByteStream.Write` call) before it is cancelled. | | `use_legacy_resource_names` | boolean | — | false | Use legacy `ByteStream` resource name format, omitting the digest function component from the path. | | `headers` | map of string to string | — | — | Static headers to attach to every outgoing gRPC request sent to this store's upstream endpoints. Useful for fixed authentication tokens (e.g. `{"authorization": "Bearer "}`) and other static metadata. | @@ -869,7 +869,7 @@ See [`experimental_cloud_object_store`](#experimental_cloud_object_store-1) for | `health_check_timeout_ms` | integer (uint64) | — | 4000 (4 seconds) | Per-call ceiling for the `check_health` PING in milliseconds. | | `read_chunk_size` | integer (uint) | — | 64KiB | The amount of data to read from the Redis server at a time. This is used to limit the amount of memory used when reading large objects from the Redis server as well as limiting the amount of time a single read operation can take. | | `connection_pool_size` | integer (uint) | — | 3 | The number of connections to keep open to the Redis servers. | -| `max_chunk_uploads_per_update` | integer (uint) | — | 10 | The maximum number of upload chunks to allow per update. This is used to limit the amount of memory used when uploading large objects to the Redis server. A good rule of thumb is to think of the data as: `AVAIL_MEMORY / (read_chunk_size * max_chunk_uploads_per_update) = THORETICAL_MAX_CONCURRENT_UPLOADS` (note: it is a good idea to divide `AVAIL_MAX_MEMORY` by ~10 to account for other memory usage) | +| `max_chunk_uploads_per_update` | integer (uint) | — | 10 | The maximum number of upload chunks to allow per update. This is used to limit the amount of memory used when uploading large objects to the Redis server. A good rule of thumb is to think of the data as: `AVAIL_MEMORY / (read_chunk_size * max_chunk_uploads_per_update) = THORETICAL_MAX_CONCURRENT_UPLOADS` (note: it's a good idea to divide `AVAIL_MAX_MEMORY` by ~10 to account for other memory usage) | | `scan_count` | integer (uint) | — | 10000 | The COUNT value passed when scanning keys in Redis. This is used to hint the amount of work that should be done per response. | | `retry` | [Retry](#retry) | — | — | Retry configuration to use when a network request fails. | | `max_client_permits` | integer (uint) | — | 500 | Maximum number of permitted actions to the Redis store at any one time This stops problems with timeouts due to many, many inflight actions | @@ -904,7 +904,7 @@ Configuration for `ExperimentalMongoDB` store. | Field | Type | Required | Default | Description | | --- | --- | --- | --- | --- | -| `name` | string | — | {Index position in the workers list} | Name of the worker. This is give a more friendly name to a worker for logging and metric publishing. This is also the prefix of the worker id (i.e., "{name}{uuidv6}"). | +| `name` | string | — | {Index position in the workers list} | Name of the worker. This is give a more friendly name to a worker for logging and metric publishing. This is also the prefix of the worker id (i.e. "{name}{uuidv6}"). | | `worker_api_endpoint` | [EndpointConfig](#endpointconfig) | Yes | — | Endpoint which the worker will connect to the scheduler's `WorkerApiService`. | | `max_action_timeout_s` | integer (uint) | — | 20 minutes | The maximum time an action is allowed to run. If a task requests for a timeout longer than this time limit, the task will be rejected. Value in seconds. | | `max_upload_timeout_s` | integer (uint) | — | 10 minutes | Maximum time allowed for uploading action results to CAS after execution completes. If upload takes longer than this, the action fails with `DeadlineExceeded` and may be retried by the scheduler. Value in seconds. | @@ -991,7 +991,7 @@ Listener for HTTP/HTTPS/HTTP2 sockets. | --- | --- | --- | --- | --- | | `cas` | array of [CasServiceConfig](#casserviceconfig) | — | — | The Content Addressable Storage (CAS) backend config. The key is the `instance_name` used in the protocol and the value is the underlying CAS store config. | | `ac` | array of [ActionCacheServiceConfig](#actioncacheserviceconfig) | — | — | The Action Cache (AC) backend config. The key is the `instance_name` used in the protocol and the value is the underlying AC store config. | -| `capabilities` | array of [CapabilitiesServiceConfig](#capabilitiesserviceconfig) | — | — | Capabilities service is required in order to use most of the bazel protocol. This service is used to provide the supported features and versions of this bazel GRPC service. | +| `capabilities` | array of [CapabilitiesServiceConfig](#capabilitiesserviceconfig) | — | — | Capabilities service is required in order to use most of the bazel protocol. This service is used to provide the supported features and versions of this bazel gRPC service. | | `execution` | array of [ExecutionServiceConfig](#executionserviceconfig) | — | — | The remote execution service configuration. NOTE: This service is under development and is currently just a place holder. | | `bytestream` | array of [ByteStreamServiceConfig](#bytestreamserviceconfig) | — | — | This is the service used to stream data to and from the CAS. Bazel's protocol strongly encourages users to use this streaming interface to interact with the CAS when the data is large. | | `fetch` | array of [FetchServiceConfig](#fetchserviceconfig) | — | — | These two are collectively the Remote Asset protocol, but it's defined as two separate services | @@ -1066,7 +1066,7 @@ The total delay is additive, so this example produces 9.525 to 15.875 s of total | `max_retries` | integer (uint) | — | `0` | Maximum number of retries until retrying stops. Setting this to zero will always attempt 1 time, but not retry. | | `delay` | number (float) | — | `0` | Delay in seconds for exponential back off. | | `jitter` | number (float) | — | `0` | Amount of jitter to add as a percentage in decimal form. This will change the formula like: | -| `retry_on_errors` | array of [ErrorCode](#errorcode) | — | — | A list of error codes to retry on, if this is not set then the default error codes to retry on are used. These default codes are the most likely to be non-permanent: `Unknown`; `Cancelled`; `DeadlineExceeded`; `ResourceExhausted`; `Aborted`; `Internal`; `Unavailable`; `DataLoss` | +| `retry_on_errors` | array of [ErrorCode](#errorcode) | — | — | A list of error codes to retry on, if this isn't set then the default error codes to retry on are used. These default codes are the most likely to be non-permanent: `Unknown`; `Cancelled`; `DeadlineExceeded`; `ResourceExhausted`; `Aborted`; `Internal`; `Unavailable`; `DataLoss` | ## ExperimentalOntapS3Spec @@ -1122,7 +1122,7 @@ Configuration for an individual shard of the store. | Field | Type | Required | Default | Description | | --- | --- | --- | --- | --- | -| `address` | string | Yes | — | The endpoint address (i.e. grpc://example.com:443 or grpcs://example.com:443). | +| `address` | string | Yes | — | The endpoint address (i.e. `grpc(s)://example.com:443`). | | `tls_config` | [ClientTlsConfig](#clienttlsconfig) | — | — | The TLS configuration to use to connect to the endpoint (if grpcs). | | `concurrency_limit` | integer (uint) | — | — | The maximum concurrency to allow on this endpoint. | | `connect_timeout_s` | integer (uint64) | — | 30 seconds | Timeout for establishing a TCP connection to the endpoint (seconds). | @@ -1203,6 +1203,8 @@ One of: | `max_entries` | integer (uint) | — | 1000 | Maximum number of cached directories. | | `max_size_bytes` | integer (uint64) | — | 10737418240 (10 GB) | Maximum total size in bytes for all cached directories (0 = unlimited). | | `cache_root` | string | — | `{work_directory}/../directory_cache` | Base directory for cache storage. This directory will be managed by the worker and should be on the same filesystem as `work_directory`. | +| `experimental_subtree_caching` | boolean | — | false (only root directories are cached; existing behavior) | Optional and experimental: additionally cache every subdirectory by its own `Directory` digest, not just the root directory. REAPI Merkle nodes are content-addressed, so a subtree that is byte-identical between two different roots has the same digest and can be materialized with a single hardlink pass instead of being rebuilt from the CAS. This makes the common "one input file changed out of thousands" case reuse every unchanged subtree. | +| `max_concurrent_fetches` | integer (uint) | — | 64 | Maximum number of concurrent slow-store fetches across ALL directory constructions of this cache. This bound protects backing stores from RPC storms: per-level construction concurrency compounds multiplicatively across tree levels and concurrent actions. | ## PropertyType diff --git a/web/apps/docs/content/docs/reference/nativelink-config/v1.0.0.mdx b/web/apps/docs/content/docs/reference/nativelink-config/v1.0.0.mdx index cc4e46aca..f63591683 100644 --- a/web/apps/docs/content/docs/reference/nativelink-config/v1.0.0.mdx +++ b/web/apps/docs/content/docs/reference/nativelink-config/v1.0.0.mdx @@ -667,7 +667,7 @@ See [`experimental_cloud_object_store`](#experimental_cloud_object_store-1) for | Field | Type | Required | Default | Description | | --- | --- | --- | --- | --- | -| `backend` | [StoreSpec](#storespec) | Yes | — | The underlying store wrap around. All content will first flow through self before forwarding to backend. In the event there is an error detected in self, the connection to the backend will be terminated, and early termination should always cause updates to fail on the backend. | +| `backend` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | The underlying store wrap around. All content will first flow through self before forwarding to backend. In the event there is an error detected in self, the connection to the backend will be terminated, and early termination should always cause updates to fail on the backend. | | `verify_size` | boolean | — | `false` | If set the store will verify the size of the data before accepting an upload of data. | | `verify_hash` | boolean | — | `false` | If the data should be hashed and verify that the key matches the computed hash. The hash function is automatically determined based request and if not set will use the global default. | @@ -675,22 +675,22 @@ See [`experimental_cloud_object_store`](#experimental_cloud_object_store-1) for | Field | Type | Required | Default | Description | | --- | --- | --- | --- | --- | -| `backend` | [StoreSpec](#storespec) | Yes | — | The underlying store that will have it's results validated before sending to client. | -| `cas_store` | [StoreSpec](#storespec) | Yes | — | When a request is made, the results are decoded and all output digests/files are verified to exist in this CAS store before returning success. | +| `backend` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | The underlying store that will have it's results validated before sending to client. | +| `cas_store` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | When a request is made, the results are decoded and all output digests/files are verified to exist in this CAS store before returning success. | ## CompressionSpec | Field | Type | Required | Default | Description | | --- | --- | --- | --- | --- | -| `backend` | [StoreSpec](#storespec) | Yes | — | The underlying store wrap around. All content will first flow through self before forwarding to backend. In the event there is an error detected in self, the connection to the backend will be terminated, and early termination should always cause updates to fail on the backend. | +| `backend` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | The underlying store wrap around. All content will first flow through self before forwarding to backend. In the event there is an error detected in self, the connection to the backend will be terminated, and early termination should always cause updates to fail on the backend. | | `compression_algorithm` | [CompressionAlgorithm](#compressionalgorithm) | Yes | — | The compression algorithm to use. | ## DedupSpec | Field | Type | Required | Default | Description | | --- | --- | --- | --- | --- | -| `index_store` | [StoreSpec](#storespec) | Yes | — | Store used to store the index of each dedup slice. This store should generally be fast and small. | -| `content_store` | [StoreSpec](#storespec) | Yes | — | The store where the individual chunks will be uploaded. This store should generally be the slower & larger store. | +| `index_store` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | Store used to store the index of each dedup slice. This store should generally be fast and small. | +| `content_store` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | The store where the individual chunks will be uploaded. This store should generally be the slower & larger store. | | `min_size` | integer (uint32) | — | 65536 (64k) | Minimum size that a chunk will be when slicing up the content. Note: This setting can be increased to improve performance because it will actually not check this number of bytes when deciding where to partition the data. | | `normal_size` | integer (uint32) | — | 262144 (256k) | A best-effort attempt will be made to keep the average size of the chunks to this number. It is not a guarantee, but a slight attempt will be made. | | `max_size` | integer (uint32) | — | 524288 (512k) | Maximum size a chunk is allowed to be. | @@ -700,16 +700,16 @@ See [`experimental_cloud_object_store`](#experimental_cloud_object_store-1) for | Field | Type | Required | Default | Description | | --- | --- | --- | --- | --- | -| `backend` | [StoreSpec](#storespec) | Yes | — | The underlying store wrap around. All content will first flow through self before forwarding to backend. In the event there is an error detected in self, the connection to the backend will be terminated, and early termination should always cause updates to fail on the backend. | +| `backend` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | The underlying store wrap around. All content will first flow through self before forwarding to backend. In the event there is an error detected in self, the connection to the backend will be terminated, and early termination should always cause updates to fail on the backend. | | `eviction_policy` | [EvictionPolicy](#evictionpolicy) | — | — | Policy used to evict items out of the store. Failure to set this value will cause items to never be removed from the store causing infinite memory usage. | ## FastSlowSpec | Field | Type | Required | Default | Description | | --- | --- | --- | --- | --- | -| `fast` | [StoreSpec](#storespec) | Yes | — | Fast store that will be attempted to be contacted before reaching out to the `slow` store. | +| `fast` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | Fast store that will be attempted to be contacted before reaching out to the `slow` store. | | `fast_direction` | [StoreDirection](#storedirection) | — | `"both"` | How to handle the fast store. This can be useful to set to Get for worker nodes such that results are persisted to the slow store only. | -| `slow` | [StoreSpec](#storespec) | Yes | — | If the object does not exist in the `fast` store it will try to get it from this store. | +| `slow` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | If the object does not exist in the `fast` store it will try to get it from this store. | | `slow_direction` | [StoreDirection](#storedirection) | — | `"both"` | How to handle the slow store. This can be useful if creating a diode and you wish to have an upstream read only store. | ## ShardSpec @@ -740,8 +740,8 @@ See [`experimental_cloud_object_store`](#experimental_cloud_object_store-1) for | Field | Type | Required | Default | Description | | --- | --- | --- | --- | --- | | `size` | integer (uint64) | Yes | — | Size to partition the data on. | -| `lower_store` | [StoreSpec](#storespec) | Yes | — | Store to send data when object is < (less than) size. | -| `upper_store` | [StoreSpec](#storespec) | Yes | — | Store to send data when object is >= (less than eq) size. | +| `lower_store` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | Store to send data when object is < (less than) size. | +| `upper_store` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | Store to send data when object is >= (less than eq) size. | ## GrpcSpec @@ -970,553 +970,6 @@ The total delay is additive, so this example produces 9.525 to 15.875 s of total | `insecure_allow_http` | boolean | — | false | Allow unencrypted HTTP connections. Only use this for local testing. | | `disable_http2` | boolean | — | false | Disable HTTP/2 connections and only use HTTP/1.1. Default client configuration will have HTTP/1.1 and HTTP/2 enabled for connection schemes. HTTP/2 should be disabled if environments have poor support or performance related to HTTP/2. Safe to keep default unless underlying network environment, S3, or GCS API servers specify otherwise. | -## StoreSpec - -Plus exactly one of the following variants (the key selects the variant): - -### `memory` - -Memory store will store all data in a hash map in memory. - -**Example JSON5 config:** -```json5 -"memory": { - "eviction_policy": { - "max_bytes": "10mb", - } -} -``` - -**Type:** [MemorySpec](#memoryspec) - -### `experimental_cloud_object_store` - -A generic blob store that will store files on the cloud -provider. This configuration will never delete files, so you are -responsible for purging old files in other ways. -It supports the following backends: - -1. **Amazon S3:** - S3 store will use Amazon's S3 service as a backend to store - the files. This configuration can be used to share files - across multiple instances. Uses system certificates for TLS - verification via `rustls-platform-verifier`. - - **Example JSON5 config:** - ```json5 - "experimental_cloud_object_store": { - "provider": "aws", - "region": "eu-north-1", - "bucket": "crossplane-bucket-af79aeca9", - "key_prefix": "test-prefix-index/", - "retry": { - "max_retries": 6, - "delay": 0.3, - "jitter": 0.5 - }, - "multipart_max_concurrent_uploads": 10 - } - ``` - -2. **Google Cloud Storage:** - GCS store uses Google's GCS service as a backend to store - the files. This configuration can be used to share files - across multiple instances. - - **Example JSON5 config:** - ```json5 - "experimental_cloud_object_store": { - "provider": "gcs", - "bucket": "test-bucket", - "key_prefix": "test-prefix-index/", - "retry": { - "max_retries": 6, - "delay": 0.3, - "jitter": 0.5 - }, - "multipart_max_concurrent_uploads": 10 - } - ``` - -3. **Azure Blob Store:** - Azure Blob store will use Microsoft's Azure Blob service as a - backend to store the files. This configuration can be used to - share files across multiple instances. - - **Example JSON5 config:** - ```json5 - "experimental_cloud_object_store": { - "provider": "azure", - "account_name": "cloudshell1393657559", - "container": "simple-test-container", - "key_prefix": "folder/", - "retry": { - "max_retries": 6, - "delay": 0.3, - "jitter": 0.5 - }, - "multipart_max_concurrent_uploads": 10 - } - ``` - -4. **`NetApp` ONTAP S3** - `NetApp` ONTAP S3 store will use ONTAP's S3-compatible storage as a backend - to store files. This store is specifically configured for ONTAP's S3 requirements - including custom TLS configuration, credentials management, and proper vserver - configuration. - - This store uses AWS environment variables for credentials: - - `AWS_ACCESS_KEY_ID` - - `AWS_SECRET_ACCESS_KEY` - - `AWS_DEFAULT_REGION` - - **Example JSON5 config:** - ```json5 - "experimental_cloud_object_store": { - "provider": "ontap", - "endpoint": "https://ontap-s3-endpoint:443", - "vserver_name": "your-vserver", - "bucket": "your-bucket", - "root_certificates": "/path/to/certs.pem", // Optional - "key_prefix": "test-prefix/", // Optional - "retry": { - "max_retries": 6, - "delay": 0.3, - "jitter": 0.5 - }, - "multipart_max_concurrent_uploads": 10 - } - ``` - -**Type:** [ExperimentalCloudObjectSpec](#experimentalcloudobjectspec) - -### `ontap_s3_existence_cache` - -ONTAP S3 Existence Cache provides a caching layer on top of the ONTAP S3 store -to optimize repeated existence checks. It maintains an in-memory cache of object -digests and periodically syncs this cache to disk for persistence. - -The cache helps reduce latency for repeated calls to check object existence, -while still ensuring eventual consistency with the underlying ONTAP S3 store. - -Example JSON5 config: -```json5 -"ontap_s3_existence_cache": { - "index_path": "/path/to/cache/index.json", - "sync_interval_seconds": 300, - "backend": { - "endpoint": "https://ontap-s3-endpoint:443", - "vserver_name": "your-vserver", - "bucket": "your-bucket", - "key_prefix": "test-prefix/" - } -} -``` - -**Type:** [OntapS3ExistenceCacheSpec](#ontaps3existencecachespec) - -### `verify` - -Verify store is used to apply verifications to an underlying -store implementation. It is strongly encouraged to validate -as much data as you can before accepting data from a client, -failing to do so may cause the data in the store to be -populated with invalid data causing all kinds of problems. - -The suggested configuration is to have the CAS validate the -hash and size and the AC validate nothing. - -**Example JSON5 config:** -```json5 -"verify": { - "backend": { - "memory": { - "eviction_policy": { - "max_bytes": "500mb" - } - }, - }, - "verify_size": true, - "verify_hash": true -} -``` - -**Type:** [VerifySpec](#verifyspec) - -### `completeness_checking` - -Completeness checking store verifies if the -output files & folders exist in the CAS before forwarding -the request to the underlying store. -Note: This store should only be used on AC stores. - -**Example JSON5 config:** -```json5 -"completeness_checking": { - "backend": { - "filesystem": { - "content_path": "~/.cache/nativelink/content_path-ac", - "temp_path": "~/.cache/nativelink/tmp_path-ac", - "eviction_policy": { - "max_bytes": "500mb", - } - } - }, - "cas_store": { - "ref_store": { - "name": "CAS_MAIN_STORE" - } - } -} -``` - -**Type:** [CompletenessCheckingSpec](#completenesscheckingspec) - -### `compression` - -A compression store that will compress the data inbound and -outbound. There will be a non-trivial cost to compress and -decompress the data, but in many cases if the final store is -a store that requires network transport and/or storage space -is a concern it is often faster and more efficient to use this -store before those stores. - -**Example JSON5 config:** -```json5 -"compression": { - "compression_algorithm": { - "lz4": {} - }, - "backend": { - "filesystem": { - "content_path": "/tmp/nativelink/data/content_path-cas", - "temp_path": "/tmp/nativelink/data/tmp_path-cas", - "eviction_policy": { - "max_bytes": "2gb", - } - } - } -} -``` - -**Type:** [CompressionSpec](#compressionspec) - -### `dedup` - -A dedup store will take the inputs and run a rolling hash -algorithm on them to slice the input into smaller parts then -run a sha256 algorithm on the slice and if the object doesn't -already exist, upload the slice to the `content_store` using -a new digest of just the slice. Once all parts exist, an -Action-Cache-like digest will be built and uploaded to the -`index_store` which will contain a reference to each -chunk/digest of the uploaded file. Downloading a request will -first grab the index from the `index_store`, and forward the -download content of each chunk as if it were one file. - -This store is exceptionally good when the following conditions -are met: -* Content is mostly the same (inserts, updates, deletes are ok) -* Content is not compressed or encrypted -* Uploading or downloading from `content_store` is the bottleneck. - -Note: This store pairs well when used with `CompressionSpec` as -the `content_store`, but never put `DedupSpec` as the backend of -`CompressionSpec` as it will negate all the gains. - -Note: When running `.has()` on this store, it will only check -to see if the entry exists in the `index_store` and not check -if the individual chunks exist in the `content_store`. - -**Example JSON5 config:** -```json5 -"dedup": { - "index_store": { - "memory": { - "eviction_policy": { - "max_bytes": "1GB", - } - } - }, - "content_store": { - "compression": { - "compression_algorithm": { - "lz4": {} - }, - "backend": { - "fast_slow": { - "fast": { - "memory": { - "eviction_policy": { - "max_bytes": "500MB", - } - } - }, - "slow": { - "filesystem": { - "content_path": "/tmp/nativelink/data/content_path-content", - "temp_path": "/tmp/nativelink/data/tmp_path-content", - "eviction_policy": { - "max_bytes": "2gb" - } - } - } - } - } - } - } -} -``` - -**Type:** [DedupSpec](#dedupspec) - -### `existence_cache` - -Existence store will wrap around another store and cache calls -to has so that subsequent `has_with_results` calls will be -faster. This is useful for cases when you have a store that -is slow to respond to has calls. -Note: This store should only be used on CAS stores. - -**Example JSON5 config:** -```json5 -"existence_cache": { - "backend": { - "memory": { - "eviction_policy": { - "max_bytes": "500mb", - } - } - }, - // Note this is the existence store policy, not the backend policy - "eviction_policy": { - "max_seconds": 100, - } -} -``` - -**Type:** [ExistenceCacheSpec](#existencecachespec) - -### `fast_slow` - -`FastSlow` store will first try to fetch the data from the `fast` -store and then if it does not exist try the `slow` store. -When the object does exist in the `slow` store, it will copy -the data to the `fast` store while returning the data. -This store should be thought of as a store that "buffers" -the data to the `fast` store. -On uploads it will mirror data to both `fast` and `slow` stores. - -WARNING: If you need data to always exist in the `slow` store -for something like remote execution, be careful because this -store will never check to see if the objects exist in the -`slow` store if it exists in the `fast` store (i.e., it assumes -that if an object exists in the `fast` store it will exist in -the `slow` store). - -***Example JSON5 config:*** -```json5 -"fast_slow": { - "fast": { - "filesystem": { - "content_path": "/tmp/nativelink/data/content_path-index", - "temp_path": "/tmp/nativelink/data/tmp_path-index", - "eviction_policy": { - "max_bytes": "500mb", - } - } - }, - "slow": { - "filesystem": { - "content_path": "/tmp/nativelink/data/content_path-index", - "temp_path": "/tmp/nativelink/data/tmp_path-index", - "eviction_policy": { - "max_bytes": "500mb", - } - } - } -} -``` - -**Type:** [FastSlowSpec](#fastslowspec) - -### `shard` - -Shards the data to multiple stores. This is useful for cases -when you want to distribute the load across multiple stores. -The digest hash is used to determine which store to send the -data to. - -**Example JSON5 config:** -```json5 -"shard": { - "stores": [ - { - "store": { - "memory": { - "eviction_policy": { - "max_bytes": "10mb" - }, - }, - }, - "weight": 1 - }] -} -``` - -**Type:** [ShardSpec](#shardspec) - -### `filesystem` - -Stores the data on the filesystem. This store is designed for -local persistent storage. Restarts of this program should restore -the previous state, meaning anything uploaded will be persistent -as long as the filesystem integrity holds. - -**Example JSON5 config:** -```json5 -"filesystem": { - "content_path": "/tmp/nativelink/data-worker-test/content_path-cas", - "temp_path": "/tmp/nativelink/data-worker-test/tmp_path-cas", - "eviction_policy": { - "max_bytes": "10gb", - } -} -``` - -**Type:** [FilesystemSpec](#filesystemspec) - -### `ref_store` - -Store used to reference a store in the root store manager. -This is useful for cases when you want to share a store in different -nested stores. Example, you may want to share the same memory store -used for the action cache, but use a `FastSlowSpec` and have the fast -store also share the memory store for efficiency. - -**Example JSON5 config:** -```json5 -"ref_store": { - "name": "FS_CONTENT_STORE" -} -``` - -**Type:** [RefSpec](#refspec) - -### `size_partitioning` - -Uses the size field of the digest to separate which store to send the -data. This is useful for cases when you'd like to put small objects -in one store and large objects in another store. This should only be -used if the size field is the real size of the content, in other -words, don't use on AC (Action Cache) stores. Any store where you can -safely use `VerifySpec.verify_size = true`, this store should be safe -to use (i.e., CAS stores). - -**Example JSON5 config:** -```json5 -"size_partitioning": { - "size": "128mib", - "lower_store": { - "memory": { - "eviction_policy": { - "max_bytes": "${NATIVELINK_CAS_MEMORY_CONTENT_LIMIT:-100mb}" - } - } - }, - "upper_store": { - /// This store discards data larger than 128mib. - "noop": {} - } -} -``` - -**Type:** [SizePartitioningSpec](#sizepartitioningspec) - -### `grpc` - -This store will pass-through calls to another GRPC store. This store -is not designed to be used as a sub-store of another store, but it -does satisfy the interface and will likely work. - -One major GOTCHA is that some stores use a special function on this -store to get the size of the underlying object, which is only reliable -when this store is serving the a CAS store, not an AC store. If using -this store directly without being a child of any store there are no -side effects and is the most efficient way to use it. - -**Example JSON5 config:** -```json5 -"grpc": { - "instance_name": "main", - "endpoints": [ - {"address": "grpc://${CAS_ENDPOINT:-127.0.0.1}:50051"} - ], - "connections_per_endpoint": "5", - "rpc_timeout_s": "5m", - "store_type": "ac" -} -``` - -**Type:** [GrpcSpec](#grpcspec) - -### `redis_store` - -Stores data in any stores compatible with Redis APIs. - -Pairs well with `SizePartitioning` and/or `FastSlow` stores. -Ideal for accepting small object sizes as most Redis store -services have a max file upload of between 256Mb-512Mb. - -**Example JSON5 config:** -```json5 -"redis_store": { - "addresses": [ - "redis://127.0.0.1:6379/", - ], - "max_client_permits": 1000, -} -``` - -**Type:** [RedisSpec](#redisspec) - -### `noop` - -Noop store is a store that sends streams into the void and all data -retrieval will return 404 (`NotFound`). This can be useful for cases -where you may need to partition your data and part of your data needs -to be discarded. - -**Example JSON5 config:** -```json5 -"noop": {} -``` - -**Type:** [NoopSpec](#noopspec) - -### `experimental_mongo` - -Experimental `MongoDB` store implementation. - -This store uses `MongoDB` as a backend for storing data. It supports -both CAS (Content Addressable Storage) and scheduler data with -optional change streams for real-time updates. - -**Example JSON5 config:** -```json5 -"experimental_mongo": { - "connection_string": "mongodb://localhost:27017", - "database": "nativelink", - "cas_collection": "cas", - "key_prefix": "cas:", - "read_chunk_size": 65536, - "max_concurrent_uploads": 10, - "enable_change_streams": false -} -``` - -**Type:** [ExperimentalMongoSpec](#experimentalmongospec) - ## CompressionAlgorithm Plus exactly one of the following variants (the key selects the variant): @@ -1548,7 +1001,7 @@ Configuration for an individual shard of the store. | Field | Type | Required | Default | Description | | --- | --- | --- | --- | --- | -| `store` | [StoreSpec](#storespec) | Yes | — | Store to shard the data to. | +| `store` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | Store to shard the data to. | | `weight` | integer (uint32) | — | 1 | The weight of the store. This is used to determine how much data should be sent to the store. The actual percentage is the sum of all the store's weights divided by the individual store's weight. | ## GrpcEndpoint diff --git a/web/apps/docs/content/docs/reference/nativelink-config/v1.1.0.mdx b/web/apps/docs/content/docs/reference/nativelink-config/v1.1.0.mdx index 410d1a6b1..7b7a0c3b6 100644 --- a/web/apps/docs/content/docs/reference/nativelink-config/v1.1.0.mdx +++ b/web/apps/docs/content/docs/reference/nativelink-config/v1.1.0.mdx @@ -677,7 +677,7 @@ See [`experimental_cloud_object_store`](#experimental_cloud_object_store-1) for | Field | Type | Required | Default | Description | | --- | --- | --- | --- | --- | -| `backend` | [StoreSpec](#storespec) | Yes | — | The underlying store wrap around. All content will first flow through self before forwarding to backend. In the event there is an error detected in self, the connection to the backend will be terminated, and early termination should always cause updates to fail on the backend. | +| `backend` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | The underlying store wrap around. All content will first flow through self before forwarding to backend. In the event there is an error detected in self, the connection to the backend will be terminated, and early termination should always cause updates to fail on the backend. | | `verify_size` | boolean | — | `false` | If set the store will verify the size of the data before accepting an upload of data. | | `verify_hash` | boolean | — | `false` | If the data should be hashed and verify that the key matches the computed hash. The hash function is automatically determined based request and if not set will use the global default. | @@ -685,22 +685,22 @@ See [`experimental_cloud_object_store`](#experimental_cloud_object_store-1) for | Field | Type | Required | Default | Description | | --- | --- | --- | --- | --- | -| `backend` | [StoreSpec](#storespec) | Yes | — | The underlying store that will have it's results validated before sending to client. | -| `cas_store` | [StoreSpec](#storespec) | Yes | — | When a request is made, the results are decoded and all output digests/files are verified to exist in this CAS store before returning success. | +| `backend` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | The underlying store that will have it's results validated before sending to client. | +| `cas_store` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | When a request is made, the results are decoded and all output digests/files are verified to exist in this CAS store before returning success. | ## CompressionSpec | Field | Type | Required | Default | Description | | --- | --- | --- | --- | --- | -| `backend` | [StoreSpec](#storespec) | Yes | — | The underlying store wrap around. All content will first flow through self before forwarding to backend. In the event there is an error detected in self, the connection to the backend will be terminated, and early termination should always cause updates to fail on the backend. | +| `backend` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | The underlying store wrap around. All content will first flow through self before forwarding to backend. In the event there is an error detected in self, the connection to the backend will be terminated, and early termination should always cause updates to fail on the backend. | | `compression_algorithm` | [CompressionAlgorithm](#compressionalgorithm) | Yes | — | The compression algorithm to use. | ## DedupSpec | Field | Type | Required | Default | Description | | --- | --- | --- | --- | --- | -| `index_store` | [StoreSpec](#storespec) | Yes | — | Store used to store the index of each dedup slice. This store should generally be fast and small. | -| `content_store` | [StoreSpec](#storespec) | Yes | — | The store where the individual chunks will be uploaded. This store should generally be the slower & larger store. | +| `index_store` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | Store used to store the index of each dedup slice. This store should generally be fast and small. | +| `content_store` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | The store where the individual chunks will be uploaded. This store should generally be the slower & larger store. | | `min_size` | integer (uint32) | — | 65536 (64k) | Minimum size that a chunk will be when slicing up the content. Note: This setting can be increased to improve performance because it will actually not check this number of bytes when deciding where to partition the data. | | `normal_size` | integer (uint32) | — | 262144 (256k) | A best-effort attempt will be made to keep the average size of the chunks to this number. It is not a guarantee, but a slight attempt will be made. | | `max_size` | integer (uint32) | — | 524288 (512k) | Maximum size a chunk is allowed to be. | @@ -710,16 +710,16 @@ See [`experimental_cloud_object_store`](#experimental_cloud_object_store-1) for | Field | Type | Required | Default | Description | | --- | --- | --- | --- | --- | -| `backend` | [StoreSpec](#storespec) | Yes | — | The underlying store wrap around. All content will first flow through self before forwarding to backend. In the event there is an error detected in self, the connection to the backend will be terminated, and early termination should always cause updates to fail on the backend. | +| `backend` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | The underlying store wrap around. All content will first flow through self before forwarding to backend. In the event there is an error detected in self, the connection to the backend will be terminated, and early termination should always cause updates to fail on the backend. | | `eviction_policy` | [EvictionPolicy](#evictionpolicy) | — | — | Policy used to evict items out of the store. Failure to set this value will cause items to never be removed from the store causing infinite memory usage. | ## FastSlowSpec | Field | Type | Required | Default | Description | | --- | --- | --- | --- | --- | -| `fast` | [StoreSpec](#storespec) | Yes | — | Fast store that will be attempted to be contacted before reaching out to the `slow` store. | +| `fast` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | Fast store that will be attempted to be contacted before reaching out to the `slow` store. | | `fast_direction` | [StoreDirection](#storedirection) | — | `"both"` | How to handle the fast store. This can be useful to set to Get for worker nodes such that results are persisted to the slow store only. | -| `slow` | [StoreSpec](#storespec) | Yes | — | If the object does not exist in the `fast` store it will try to get it from this store. | +| `slow` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | If the object does not exist in the `fast` store it will try to get it from this store. | | `slow_direction` | [StoreDirection](#storedirection) | — | `"both"` | How to handle the slow store. This can be useful if creating a diode and you wish to have an upstream read only store. | ## ShardSpec @@ -750,8 +750,8 @@ See [`experimental_cloud_object_store`](#experimental_cloud_object_store-1) for | Field | Type | Required | Default | Description | | --- | --- | --- | --- | --- | | `size` | integer (uint64) | Yes | — | Size to partition the data on. | -| `lower_store` | [StoreSpec](#storespec) | Yes | — | Store to send data when object is < (less than) size. | -| `upper_store` | [StoreSpec](#storespec) | Yes | — | Store to send data when object is >= (less than eq) size. | +| `lower_store` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | Store to send data when object is < (less than) size. | +| `upper_store` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | Store to send data when object is >= (less than eq) size. | ## GrpcSpec @@ -986,563 +986,6 @@ The total delay is additive, so this example produces 9.525 to 15.875 s of total | `insecure_allow_http` | boolean | — | false | Allow unencrypted HTTP connections. Only use this for local testing. | | `disable_http2` | boolean | — | false | Disable HTTP/2 connections and only use HTTP/1.1. Default client configuration will have HTTP/1.1 and HTTP/2 enabled for connection schemes. HTTP/2 should be disabled if environments have poor support or performance related to HTTP/2. Safe to keep default unless underlying network environment, S3, or GCS API servers specify otherwise. | -## StoreSpec - -Plus exactly one of the following variants (the key selects the variant): - -### `memory` - -Memory store will store all data in a hash map in memory. - -**Example JSON5 config:** -```json5 -"memory": { - "eviction_policy": { - "max_bytes": "10mb", - } -} -``` - -**Type:** [MemorySpec](#memoryspec) - -### `experimental_cloud_object_store` - -A generic blob store that will store files on the cloud -provider. This configuration will never delete files, so you are -responsible for purging old files in other ways. -It supports the following backends: - -1. **Amazon S3:** - S3 store will use Amazon's S3 service as a backend to store - the files. This configuration can be used to share files - across multiple instances. Uses system certificates for TLS - verification via `rustls-platform-verifier`. - - **Example JSON5 config:** - ```json5 - "experimental_cloud_object_store": { - "provider": "aws", - "region": "eu-north-1", - "bucket": "crossplane-bucket-af79aeca9", - "key_prefix": "test-prefix-index/", - "retry": { - "max_retries": 6, - "delay": 0.3, - "jitter": 0.5 - }, - "multipart_max_concurrent_uploads": 10 - } - ``` - -2. **Google Cloud Storage:** - GCS store uses Google's GCS service as a backend to store - the files. This configuration can be used to share files - across multiple instances. - - **Example JSON5 config:** - ```json5 - "experimental_cloud_object_store": { - "provider": "gcs", - "bucket": "test-bucket", - "key_prefix": "test-prefix-index/", - "retry": { - "max_retries": 6, - "delay": 0.3, - "jitter": 0.5 - }, - "multipart_max_concurrent_uploads": 10 - } - ``` - -3. **Azure Blob Store:** - Azure Blob store will use Microsoft's Azure Blob service as a - backend to store the files. This configuration can be used to - share files across multiple instances. - - **Example JSON5 config:** - ```json5 - "experimental_cloud_object_store": { - "provider": "azure", - "account_name": "cloudshell1393657559", - "container": "simple-test-container", - "key_prefix": "folder/", - "retry": { - "max_retries": 6, - "delay": 0.3, - "jitter": 0.5 - }, - "multipart_max_concurrent_uploads": 10 - } - ``` - -4. **`NetApp` ONTAP S3** - `NetApp` ONTAP S3 store will use ONTAP's S3-compatible storage as a backend - to store files. This store is specifically configured for ONTAP's S3 requirements - including custom TLS configuration, credentials management, and proper vserver - configuration. - - This store uses AWS environment variables for credentials: - - `AWS_ACCESS_KEY_ID` - - `AWS_SECRET_ACCESS_KEY` - - `AWS_DEFAULT_REGION` - - **Example JSON5 config:** - ```json5 - "experimental_cloud_object_store": { - "provider": "ontap", - "endpoint": "https://ontap-s3-endpoint:443", - "vserver_name": "your-vserver", - "bucket": "your-bucket", - "root_certificates": "/path/to/certs.pem", // Optional - "key_prefix": "test-prefix/", // Optional - "retry": { - "max_retries": 6, - "delay": 0.3, - "jitter": 0.5 - }, - "multipart_max_concurrent_uploads": 10 - } - ``` - -**Type:** [ExperimentalCloudObjectSpec](#experimentalcloudobjectspec) - -### `ontap_s3_existence_cache` - -ONTAP S3 Existence Cache provides a caching layer on top of the ONTAP S3 store -to optimize repeated existence checks. It maintains an in-memory cache of object -digests and periodically syncs this cache to disk for persistence. - -The cache helps reduce latency for repeated calls to check object existence, -while still ensuring eventual consistency with the underlying ONTAP S3 store. - -Example JSON5 config: -```json5 -"ontap_s3_existence_cache": { - "index_path": "/path/to/cache/index.json", - "sync_interval_seconds": 300, - "backend": { - "endpoint": "https://ontap-s3-endpoint:443", - "vserver_name": "your-vserver", - "bucket": "your-bucket", - "key_prefix": "test-prefix/" - } -} -``` - -**Type:** [OntapS3ExistenceCacheSpec](#ontaps3existencecachespec) - -### `verify` - -Verify store is used to apply verifications to an underlying -store implementation. It is strongly encouraged to validate -as much data as you can before accepting data from a client, -failing to do so may cause the data in the store to be -populated with invalid data causing all kinds of problems. - -The suggested configuration is to have the CAS validate the -hash and size and the AC validate nothing. - -**Example JSON5 config:** -```json5 -"verify": { - "backend": { - "memory": { - "eviction_policy": { - "max_bytes": "500mb" - } - }, - }, - "verify_size": true, - "verify_hash": true -} -``` - -**Type:** [VerifySpec](#verifyspec) - -### `completeness_checking` - -Completeness checking store verifies if the -output files & folders exist in the CAS before forwarding -the request to the underlying store. -Note: This store should only be used on AC stores. - -**Example JSON5 config:** -```json5 -"completeness_checking": { - "backend": { - "filesystem": { - "content_path": "~/.cache/nativelink/content_path-ac", - "temp_path": "~/.cache/nativelink/tmp_path-ac", - "eviction_policy": { - "max_bytes": "500mb", - } - } - }, - "cas_store": { - "ref_store": { - "name": "CAS_MAIN_STORE" - } - } -} -``` - -**Type:** [CompletenessCheckingSpec](#completenesscheckingspec) - -### `compression` - -A compression store that will compress the data inbound and -outbound. There will be a non-trivial cost to compress and -decompress the data, but in many cases if the final store is -a store that requires network transport and/or storage space -is a concern it is often faster and more efficient to use this -store before those stores. - -**Example JSON5 config:** -```json5 -"compression": { - "compression_algorithm": { - "lz4": {} - }, - "backend": { - "filesystem": { - "content_path": "/tmp/nativelink/data/content_path-cas", - "temp_path": "/tmp/nativelink/data/tmp_path-cas", - "eviction_policy": { - "max_bytes": "2gb", - } - } - } -} -``` - -**Type:** [CompressionSpec](#compressionspec) - -### `dedup` - -A dedup store will take the inputs and run a rolling hash -algorithm on them to slice the input into smaller parts then -run a sha256 algorithm on the slice and if the object doesn't -already exist, upload the slice to the `content_store` using -a new digest of just the slice. Once all parts exist, an -Action-Cache-like digest will be built and uploaded to the -`index_store` which will contain a reference to each -chunk/digest of the uploaded file. Downloading a request will -first grab the index from the `index_store`, and forward the -download content of each chunk as if it were one file. - -This store is exceptionally good when the following conditions -are met: -* Content is mostly the same (inserts, updates, deletes are ok) -* Content is not compressed or encrypted -* Uploading or downloading from `content_store` is the bottleneck. - -Note: This store pairs well when used with `CompressionSpec` as -the `content_store`, but never put `DedupSpec` as the backend of -`CompressionSpec` as it will negate all the gains. - -Note: When running `.has()` on this store, it will only check -to see if the entry exists in the `index_store` and not check -if the individual chunks exist in the `content_store`. - -**Example JSON5 config:** -```json5 -"dedup": { - "index_store": { - "memory": { - "eviction_policy": { - "max_bytes": "1GB", - } - } - }, - "content_store": { - "compression": { - "compression_algorithm": { - "lz4": {} - }, - "backend": { - "fast_slow": { - "fast": { - "memory": { - "eviction_policy": { - "max_bytes": "500MB", - } - } - }, - "slow": { - "filesystem": { - "content_path": "/tmp/nativelink/data/content_path-content", - "temp_path": "/tmp/nativelink/data/tmp_path-content", - "eviction_policy": { - "max_bytes": "2gb" - } - } - } - } - } - } - } -} -``` - -**Type:** [DedupSpec](#dedupspec) - -### `existence_cache` - -Existence store will wrap around another store and cache calls -to has so that subsequent `has_with_results` calls will be -faster. This is useful for cases when you have a store that -is slow to respond to has calls. -Note: This store should only be used on CAS stores. - -**Example JSON5 config:** -```json5 -"existence_cache": { - "backend": { - "memory": { - "eviction_policy": { - "max_bytes": "500mb", - } - } - }, - // Note this is the existence store policy, not the backend policy - "eviction_policy": { - "max_seconds": 100, - } -} -``` - -**Type:** [ExistenceCacheSpec](#existencecachespec) - -### `fast_slow` - -`FastSlow` store will first try to fetch the data from the `fast` -store and then if it does not exist try the `slow` store. -When the object does exist in the `slow` store, it will copy -the data to the `fast` store while returning the data. -This store should be thought of as a store that "buffers" -the data to the `fast` store. -On uploads it will mirror data to both `fast` and `slow` stores. - -WARNING: If you need data to always exist in the `slow` store -for something like remote execution, be careful because this -store will never check to see if the objects exist in the -`slow` store if it exists in the `fast` store (i.e., it assumes -that if an object exists in the `fast` store it will exist in -the `slow` store). - -***Example JSON5 config:*** -```json5 -"fast_slow": { - "fast": { - "filesystem": { - "content_path": "/tmp/nativelink/data/content_path-index", - "temp_path": "/tmp/nativelink/data/tmp_path-index", - "eviction_policy": { - "max_bytes": "500mb", - } - } - }, - "slow": { - "filesystem": { - "content_path": "/tmp/nativelink/data/content_path-index", - "temp_path": "/tmp/nativelink/data/tmp_path-index", - "eviction_policy": { - "max_bytes": "500mb", - } - } - } -} -``` - -**Type:** [FastSlowSpec](#fastslowspec) - -### `shard` - -Shards the data to multiple stores. This is useful for cases -when you want to distribute the load across multiple stores. -The digest hash is used to determine which store to send the -data to. - -**Example JSON5 config:** -```json5 -"shard": { - "stores": [ - { - "store": { - "memory": { - "eviction_policy": { - "max_bytes": "10mb" - }, - }, - }, - "weight": 1 - }] -} -``` - -**Type:** [ShardSpec](#shardspec) - -### `filesystem` - -Stores the data on the filesystem. This store is designed for -local persistent storage. Restarts of this program should restore -the previous state, meaning anything uploaded will be persistent -as long as the filesystem integrity holds. - -**Example JSON5 config:** -```json5 -"filesystem": { - "content_path": "/tmp/nativelink/data-worker-test/content_path-cas", - "temp_path": "/tmp/nativelink/data-worker-test/tmp_path-cas", - "eviction_policy": { - "max_bytes": "10gb", - } -} -``` - -**Type:** [FilesystemSpec](#filesystemspec) - -### `ref_store` - -Store used to reference a store in the root store manager. -This is useful for cases when you want to share a store in different -nested stores. Example, you may want to share the same memory store -used for the action cache, but use a `FastSlowSpec` and have the fast -store also share the memory store for efficiency. - -**Example JSON5 config:** -```json5 -"ref_store": { - "name": "FS_CONTENT_STORE" -} -``` - -**Type:** [RefSpec](#refspec) - -### `size_partitioning` - -Uses the size field of the digest to separate which store to send the -data. This is useful for cases when you'd like to put small objects -in one store and large objects in another store. This should only be -used if the size field is the real size of the content, in other -words, don't use on AC (Action Cache) stores. Any store where you can -safely use `VerifySpec.verify_size = true`, this store should be safe -to use (i.e., CAS stores). - -**Example JSON5 config:** -```json5 -"size_partitioning": { - "size": "128mib", - "lower_store": { - "memory": { - "eviction_policy": { - "max_bytes": "${NATIVELINK_CAS_MEMORY_CONTENT_LIMIT:-100mb}" - } - } - }, - "upper_store": { - /// This store discards data larger than 128mib. - "noop": {} - } -} -``` - -**Type:** [SizePartitioningSpec](#sizepartitioningspec) - -### `grpc` - -This store will pass-through calls to another GRPC store. This store -is not designed to be used as a sub-store of another store, but it -does satisfy the interface and will likely work. - -One major GOTCHA is that some stores use a special function on this -store to get the size of the underlying object, which is only reliable -when this store is serving the a CAS store, not an AC store. If using -this store directly without being a child of any store there are no -side effects and is the most efficient way to use it. - -**Example JSON5 config:** -```json5 -"grpc": { - "instance_name": "main", - "endpoints": [ - {"address": "grpc://${CAS_ENDPOINT:-127.0.0.1}:50051"} - ], - "connections_per_endpoint": "5", - "rpc_timeout_s": "5m", - "store_type": "ac", - // Static headers attached to every outgoing request to the upstream - // remote cache. Useful for fixed service-account credentials. - "headers": { - "authorization": "Bearer my-static-token" - }, - // Header names to copy from the inbound client request and forward to - // the upstream remote cache. Use this to pass through dynamic - // credentials such as a JWT sent by the build client. - "forward_headers": ["authorization", "x-custom-token"] -} -``` - -**Type:** [GrpcSpec](#grpcspec) - -### `redis_store` - -Stores data in any stores compatible with Redis APIs. - -Pairs well with `SizePartitioning` and/or `FastSlow` stores. -Ideal for accepting small object sizes as most Redis store -services have a max file upload of between 256Mb-512Mb. - -**Example JSON5 config:** -```json5 -"redis_store": { - "addresses": [ - "redis://127.0.0.1:6379/", - ], - "max_client_permits": 1000, -} -``` - -**Type:** [RedisSpec](#redisspec) - -### `noop` - -Noop store is a store that sends streams into the void and all data -retrieval will return 404 (`NotFound`). This can be useful for cases -where you may need to partition your data and part of your data needs -to be discarded. - -**Example JSON5 config:** -```json5 -"noop": {} -``` - -**Type:** [NoopSpec](#noopspec) - -### `experimental_mongo` - -Experimental `MongoDB` store implementation. - -This store uses `MongoDB` as a backend for storing data. It supports -both CAS (Content Addressable Storage) and scheduler data with -optional change streams for real-time updates. - -**Example JSON5 config:** -```json5 -"experimental_mongo": { - "connection_string": "mongodb://localhost:27017", - "database": "nativelink", - "cas_collection": "cas", - "key_prefix": "cas:", - "read_chunk_size": 65536, - "max_concurrent_uploads": 10, - "enable_change_streams": false, - "max_requests": "100" -} -``` - -**Type:** [ExperimentalMongoSpec](#experimentalmongospec) - ## CompressionAlgorithm Plus exactly one of the following variants (the key selects the variant): @@ -1574,7 +1017,7 @@ Configuration for an individual shard of the store. | Field | Type | Required | Default | Description | | --- | --- | --- | --- | --- | -| `store` | [StoreSpec](#storespec) | Yes | — | Store to shard the data to. | +| `store` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | Store to shard the data to. | | `weight` | integer (uint32) | — | 1 | The weight of the store. This is used to determine how much data should be sent to the store. The actual percentage is the sum of all the store's weights divided by the individual store's weight. | ## GrpcEndpoint diff --git a/web/apps/docs/content/docs/reference/nativelink-config/v1.2.0.mdx b/web/apps/docs/content/docs/reference/nativelink-config/v1.2.0.mdx index b25c586b0..5877a08d8 100644 --- a/web/apps/docs/content/docs/reference/nativelink-config/v1.2.0.mdx +++ b/web/apps/docs/content/docs/reference/nativelink-config/v1.2.0.mdx @@ -677,7 +677,7 @@ See [`experimental_cloud_object_store`](#experimental_cloud_object_store-1) for | Field | Type | Required | Default | Description | | --- | --- | --- | --- | --- | -| `backend` | [StoreSpec](#storespec) | Yes | — | The underlying store wrap around. All content will first flow through self before forwarding to backend. In the event there is an error detected in self, the connection to the backend will be terminated, and early termination should always cause updates to fail on the backend. | +| `backend` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | The underlying store wrap around. All content will first flow through self before forwarding to backend. In the event there is an error detected in self, the connection to the backend will be terminated, and early termination should always cause updates to fail on the backend. | | `verify_size` | boolean | — | `false` | If set the store will verify the size of the data before accepting an upload of data. | | `verify_hash` | boolean | — | `false` | If the data should be hashed and verify that the key matches the computed hash. The hash function is automatically determined based request and if not set will use the global default. | @@ -685,22 +685,22 @@ See [`experimental_cloud_object_store`](#experimental_cloud_object_store-1) for | Field | Type | Required | Default | Description | | --- | --- | --- | --- | --- | -| `backend` | [StoreSpec](#storespec) | Yes | — | The underlying store that will have it's results validated before sending to client. | -| `cas_store` | [StoreSpec](#storespec) | Yes | — | When a request is made, the results are decoded and all output digests/files are verified to exist in this CAS store before returning success. | +| `backend` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | The underlying store that will have it's results validated before sending to client. | +| `cas_store` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | When a request is made, the results are decoded and all output digests/files are verified to exist in this CAS store before returning success. | ## CompressionSpec | Field | Type | Required | Default | Description | | --- | --- | --- | --- | --- | -| `backend` | [StoreSpec](#storespec) | Yes | — | The underlying store wrap around. All content will first flow through self before forwarding to backend. In the event there is an error detected in self, the connection to the backend will be terminated, and early termination should always cause updates to fail on the backend. | +| `backend` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | The underlying store wrap around. All content will first flow through self before forwarding to backend. In the event there is an error detected in self, the connection to the backend will be terminated, and early termination should always cause updates to fail on the backend. | | `compression_algorithm` | [CompressionAlgorithm](#compressionalgorithm) | Yes | — | The compression algorithm to use. | ## DedupSpec | Field | Type | Required | Default | Description | | --- | --- | --- | --- | --- | -| `index_store` | [StoreSpec](#storespec) | Yes | — | Store used to store the index of each dedup slice. This store should generally be fast and small. | -| `content_store` | [StoreSpec](#storespec) | Yes | — | The store where the individual chunks will be uploaded. This store should generally be the slower & larger store. | +| `index_store` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | Store used to store the index of each dedup slice. This store should generally be fast and small. | +| `content_store` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | The store where the individual chunks will be uploaded. This store should generally be the slower & larger store. | | `min_size` | integer (uint32) | — | 65536 (64k) | Minimum size that a chunk will be when slicing up the content. Note: This setting can be increased to improve performance because it will actually not check this number of bytes when deciding where to partition the data. | | `normal_size` | integer (uint32) | — | 262144 (256k) | A best-effort attempt will be made to keep the average size of the chunks to this number. It is not a guarantee, but a slight attempt will be made. | | `max_size` | integer (uint32) | — | 524288 (512k) | Maximum size a chunk is allowed to be. | @@ -710,16 +710,16 @@ See [`experimental_cloud_object_store`](#experimental_cloud_object_store-1) for | Field | Type | Required | Default | Description | | --- | --- | --- | --- | --- | -| `backend` | [StoreSpec](#storespec) | Yes | — | The underlying store wrap around. All content will first flow through self before forwarding to backend. In the event there is an error detected in self, the connection to the backend will be terminated, and early termination should always cause updates to fail on the backend. | +| `backend` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | The underlying store wrap around. All content will first flow through self before forwarding to backend. In the event there is an error detected in self, the connection to the backend will be terminated, and early termination should always cause updates to fail on the backend. | | `eviction_policy` | [EvictionPolicy](#evictionpolicy) | — | — | Policy used to evict items out of the store. Failure to set this value will cause items to never be removed from the store causing infinite memory usage. | ## FastSlowSpec | Field | Type | Required | Default | Description | | --- | --- | --- | --- | --- | -| `fast` | [StoreSpec](#storespec) | Yes | — | Fast store that will be attempted to be contacted before reaching out to the `slow` store. | +| `fast` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | Fast store that will be attempted to be contacted before reaching out to the `slow` store. | | `fast_direction` | [StoreDirection](#storedirection) | — | `"both"` | How to handle the fast store. This can be useful to set to Get for worker nodes such that results are persisted to the slow store only. | -| `slow` | [StoreSpec](#storespec) | Yes | — | If the object does not exist in the `fast` store it will try to get it from this store. | +| `slow` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | If the object does not exist in the `fast` store it will try to get it from this store. | | `slow_direction` | [StoreDirection](#storedirection) | — | `"both"` | How to handle the slow store. This can be useful if creating a diode and you wish to have an upstream read only store. | ## ShardSpec @@ -750,8 +750,8 @@ See [`experimental_cloud_object_store`](#experimental_cloud_object_store-1) for | Field | Type | Required | Default | Description | | --- | --- | --- | --- | --- | | `size` | integer (uint64) | Yes | — | Size to partition the data on. | -| `lower_store` | [StoreSpec](#storespec) | Yes | — | Store to send data when object is < (less than) size. | -| `upper_store` | [StoreSpec](#storespec) | Yes | — | Store to send data when object is >= (less than eq) size. | +| `lower_store` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | Store to send data when object is < (less than) size. | +| `upper_store` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | Store to send data when object is >= (less than eq) size. | ## GrpcSpec @@ -986,563 +986,6 @@ The total delay is additive, so this example produces 9.525 to 15.875 s of total | `insecure_allow_http` | boolean | — | false | Allow unencrypted HTTP connections. Only use this for local testing. | | `disable_http2` | boolean | — | false | Disable HTTP/2 connections and only use HTTP/1.1. Default client configuration will have HTTP/1.1 and HTTP/2 enabled for connection schemes. HTTP/2 should be disabled if environments have poor support or performance related to HTTP/2. Safe to keep default unless underlying network environment, S3, or GCS API servers specify otherwise. | -## StoreSpec - -Plus exactly one of the following variants (the key selects the variant): - -### `memory` - -Memory store will store all data in a hash map in memory. - -**Example JSON5 config:** -```json5 -"memory": { - "eviction_policy": { - "max_bytes": "10mb", - } -} -``` - -**Type:** [MemorySpec](#memoryspec) - -### `experimental_cloud_object_store` - -A generic blob store that will store files on the cloud -provider. This configuration will never delete files, so you are -responsible for purging old files in other ways. -It supports the following backends: - -1. **Amazon S3:** - S3 store will use Amazon's S3 service as a backend to store - the files. This configuration can be used to share files - across multiple instances. Uses system certificates for TLS - verification via `rustls-platform-verifier`. - - **Example JSON5 config:** - ```json5 - "experimental_cloud_object_store": { - "provider": "aws", - "region": "eu-north-1", - "bucket": "crossplane-bucket-af79aeca9", - "key_prefix": "test-prefix-index/", - "retry": { - "max_retries": 6, - "delay": 0.3, - "jitter": 0.5 - }, - "multipart_max_concurrent_uploads": 10 - } - ``` - -2. **Google Cloud Storage:** - GCS store uses Google's GCS service as a backend to store - the files. This configuration can be used to share files - across multiple instances. - - **Example JSON5 config:** - ```json5 - "experimental_cloud_object_store": { - "provider": "gcs", - "bucket": "test-bucket", - "key_prefix": "test-prefix-index/", - "retry": { - "max_retries": 6, - "delay": 0.3, - "jitter": 0.5 - }, - "multipart_max_concurrent_uploads": 10 - } - ``` - -3. **Azure Blob Store:** - Azure Blob store will use Microsoft's Azure Blob service as a - backend to store the files. This configuration can be used to - share files across multiple instances. - - **Example JSON5 config:** - ```json5 - "experimental_cloud_object_store": { - "provider": "azure", - "account_name": "cloudshell1393657559", - "container": "simple-test-container", - "key_prefix": "folder/", - "retry": { - "max_retries": 6, - "delay": 0.3, - "jitter": 0.5 - }, - "multipart_max_concurrent_uploads": 10 - } - ``` - -4. **`NetApp` ONTAP S3** - `NetApp` ONTAP S3 store will use ONTAP's S3-compatible storage as a backend - to store files. This store is specifically configured for ONTAP's S3 requirements - including custom TLS configuration, credentials management, and proper vserver - configuration. - - This store uses AWS environment variables for credentials: - - `AWS_ACCESS_KEY_ID` - - `AWS_SECRET_ACCESS_KEY` - - `AWS_DEFAULT_REGION` - - **Example JSON5 config:** - ```json5 - "experimental_cloud_object_store": { - "provider": "ontap", - "endpoint": "https://ontap-s3-endpoint:443", - "vserver_name": "your-vserver", - "bucket": "your-bucket", - "root_certificates": "/path/to/certs.pem", // Optional - "key_prefix": "test-prefix/", // Optional - "retry": { - "max_retries": 6, - "delay": 0.3, - "jitter": 0.5 - }, - "multipart_max_concurrent_uploads": 10 - } - ``` - -**Type:** [ExperimentalCloudObjectSpec](#experimentalcloudobjectspec) - -### `ontap_s3_existence_cache` - -ONTAP S3 Existence Cache provides a caching layer on top of the ONTAP S3 store -to optimize repeated existence checks. It maintains an in-memory cache of object -digests and periodically syncs this cache to disk for persistence. - -The cache helps reduce latency for repeated calls to check object existence, -while still ensuring eventual consistency with the underlying ONTAP S3 store. - -Example JSON5 config: -```json5 -"ontap_s3_existence_cache": { - "index_path": "/path/to/cache/index.json", - "sync_interval_seconds": 300, - "backend": { - "endpoint": "https://ontap-s3-endpoint:443", - "vserver_name": "your-vserver", - "bucket": "your-bucket", - "key_prefix": "test-prefix/" - } -} -``` - -**Type:** [OntapS3ExistenceCacheSpec](#ontaps3existencecachespec) - -### `verify` - -Verify store is used to apply verifications to an underlying -store implementation. It is strongly encouraged to validate -as much data as you can before accepting data from a client, -failing to do so may cause the data in the store to be -populated with invalid data causing all kinds of problems. - -The suggested configuration is to have the CAS validate the -hash and size and the AC validate nothing. - -**Example JSON5 config:** -```json5 -"verify": { - "backend": { - "memory": { - "eviction_policy": { - "max_bytes": "500mb" - } - }, - }, - "verify_size": true, - "verify_hash": true -} -``` - -**Type:** [VerifySpec](#verifyspec) - -### `completeness_checking` - -Completeness checking store verifies if the -output files & folders exist in the CAS before forwarding -the request to the underlying store. -Note: This store should only be used on AC stores. - -**Example JSON5 config:** -```json5 -"completeness_checking": { - "backend": { - "filesystem": { - "content_path": "~/.cache/nativelink/content_path-ac", - "temp_path": "~/.cache/nativelink/tmp_path-ac", - "eviction_policy": { - "max_bytes": "500mb", - } - } - }, - "cas_store": { - "ref_store": { - "name": "CAS_MAIN_STORE" - } - } -} -``` - -**Type:** [CompletenessCheckingSpec](#completenesscheckingspec) - -### `compression` - -A compression store that will compress the data inbound and -outbound. There will be a non-trivial cost to compress and -decompress the data, but in many cases if the final store is -a store that requires network transport and/or storage space -is a concern it is often faster and more efficient to use this -store before those stores. - -**Example JSON5 config:** -```json5 -"compression": { - "compression_algorithm": { - "lz4": {} - }, - "backend": { - "filesystem": { - "content_path": "/tmp/nativelink/data/content_path-cas", - "temp_path": "/tmp/nativelink/data/tmp_path-cas", - "eviction_policy": { - "max_bytes": "2gb", - } - } - } -} -``` - -**Type:** [CompressionSpec](#compressionspec) - -### `dedup` - -A dedup store will take the inputs and run a rolling hash -algorithm on them to slice the input into smaller parts then -run a sha256 algorithm on the slice and if the object doesn't -already exist, upload the slice to the `content_store` using -a new digest of just the slice. Once all parts exist, an -Action-Cache-like digest will be built and uploaded to the -`index_store` which will contain a reference to each -chunk/digest of the uploaded file. Downloading a request will -first grab the index from the `index_store`, and forward the -download content of each chunk as if it were one file. - -This store is exceptionally good when the following conditions -are met: -* Content is mostly the same (inserts, updates, deletes are ok) -* Content is not compressed or encrypted -* Uploading or downloading from `content_store` is the bottleneck. - -Note: This store pairs well when used with `CompressionSpec` as -the `content_store`, but never put `DedupSpec` as the backend of -`CompressionSpec` as it will negate all the gains. - -Note: When running `.has()` on this store, it will only check -to see if the entry exists in the `index_store` and not check -if the individual chunks exist in the `content_store`. - -**Example JSON5 config:** -```json5 -"dedup": { - "index_store": { - "memory": { - "eviction_policy": { - "max_bytes": "1GB", - } - } - }, - "content_store": { - "compression": { - "compression_algorithm": { - "lz4": {} - }, - "backend": { - "fast_slow": { - "fast": { - "memory": { - "eviction_policy": { - "max_bytes": "500MB", - } - } - }, - "slow": { - "filesystem": { - "content_path": "/tmp/nativelink/data/content_path-content", - "temp_path": "/tmp/nativelink/data/tmp_path-content", - "eviction_policy": { - "max_bytes": "2gb" - } - } - } - } - } - } - } -} -``` - -**Type:** [DedupSpec](#dedupspec) - -### `existence_cache` - -Existence store will wrap around another store and cache calls -to has so that subsequent `has_with_results` calls will be -faster. This is useful for cases when you have a store that -is slow to respond to has calls. -Note: This store should only be used on CAS stores. - -**Example JSON5 config:** -```json5 -"existence_cache": { - "backend": { - "memory": { - "eviction_policy": { - "max_bytes": "500mb", - } - } - }, - // Note this is the existence store policy, not the backend policy - "eviction_policy": { - "max_seconds": 100, - } -} -``` - -**Type:** [ExistenceCacheSpec](#existencecachespec) - -### `fast_slow` - -`FastSlow` store will first try to fetch the data from the `fast` -store and then if it does not exist try the `slow` store. -When the object does exist in the `slow` store, it will copy -the data to the `fast` store while returning the data. -This store should be thought of as a store that "buffers" -the data to the `fast` store. -On uploads it will mirror data to both `fast` and `slow` stores. - -WARNING: If you need data to always exist in the `slow` store -for something like remote execution, be careful because this -store will never check to see if the objects exist in the -`slow` store if it exists in the `fast` store (i.e., it assumes -that if an object exists in the `fast` store it will exist in -the `slow` store). - -***Example JSON5 config:*** -```json5 -"fast_slow": { - "fast": { - "filesystem": { - "content_path": "/tmp/nativelink/data/content_path-index", - "temp_path": "/tmp/nativelink/data/tmp_path-index", - "eviction_policy": { - "max_bytes": "500mb", - } - } - }, - "slow": { - "filesystem": { - "content_path": "/tmp/nativelink/data/content_path-index", - "temp_path": "/tmp/nativelink/data/tmp_path-index", - "eviction_policy": { - "max_bytes": "500mb", - } - } - } -} -``` - -**Type:** [FastSlowSpec](#fastslowspec) - -### `shard` - -Shards the data to multiple stores. This is useful for cases -when you want to distribute the load across multiple stores. -The digest hash is used to determine which store to send the -data to. - -**Example JSON5 config:** -```json5 -"shard": { - "stores": [ - { - "store": { - "memory": { - "eviction_policy": { - "max_bytes": "10mb" - }, - }, - }, - "weight": 1 - }] -} -``` - -**Type:** [ShardSpec](#shardspec) - -### `filesystem` - -Stores the data on the filesystem. This store is designed for -local persistent storage. Restarts of this program should restore -the previous state, meaning anything uploaded will be persistent -as long as the filesystem integrity holds. - -**Example JSON5 config:** -```json5 -"filesystem": { - "content_path": "/tmp/nativelink/data-worker-test/content_path-cas", - "temp_path": "/tmp/nativelink/data-worker-test/tmp_path-cas", - "eviction_policy": { - "max_bytes": "10gb", - } -} -``` - -**Type:** [FilesystemSpec](#filesystemspec) - -### `ref_store` - -Store used to reference a store in the root store manager. -This is useful for cases when you want to share a store in different -nested stores. Example, you may want to share the same memory store -used for the action cache, but use a `FastSlowSpec` and have the fast -store also share the memory store for efficiency. - -**Example JSON5 config:** -```json5 -"ref_store": { - "name": "FS_CONTENT_STORE" -} -``` - -**Type:** [RefSpec](#refspec) - -### `size_partitioning` - -Uses the size field of the digest to separate which store to send the -data. This is useful for cases when you'd like to put small objects -in one store and large objects in another store. This should only be -used if the size field is the real size of the content, in other -words, don't use on AC (Action Cache) stores. Any store where you can -safely use `VerifySpec.verify_size = true`, this store should be safe -to use (i.e., CAS stores). - -**Example JSON5 config:** -```json5 -"size_partitioning": { - "size": "128mib", - "lower_store": { - "memory": { - "eviction_policy": { - "max_bytes": "${NATIVELINK_CAS_MEMORY_CONTENT_LIMIT:-100mb}" - } - } - }, - "upper_store": { - /// This store discards data larger than 128mib. - "noop": {} - } -} -``` - -**Type:** [SizePartitioningSpec](#sizepartitioningspec) - -### `grpc` - -This store will pass-through calls to another GRPC store. This store -is not designed to be used as a sub-store of another store, but it -does satisfy the interface and will likely work. - -One major GOTCHA is that some stores use a special function on this -store to get the size of the underlying object, which is only reliable -when this store is serving the a CAS store, not an AC store. If using -this store directly without being a child of any store there are no -side effects and is the most efficient way to use it. - -**Example JSON5 config:** -```json5 -"grpc": { - "instance_name": "main", - "endpoints": [ - {"address": "grpc://${CAS_ENDPOINT:-127.0.0.1}:50051"} - ], - "connections_per_endpoint": "5", - "rpc_timeout_s": "5m", - "store_type": "ac", - // Static headers attached to every outgoing request to the upstream - // remote cache. Useful for fixed service-account credentials. - "headers": { - "authorization": "Bearer my-static-token" - }, - // Header names to copy from the inbound client request and forward to - // the upstream remote cache. Use this to pass through dynamic - // credentials such as a JWT sent by the build client. - "forward_headers": ["authorization", "x-custom-token"] -} -``` - -**Type:** [GrpcSpec](#grpcspec) - -### `redis_store` - -Stores data in any stores compatible with Redis APIs. - -Pairs well with `SizePartitioning` and/or `FastSlow` stores. -Ideal for accepting small object sizes as most Redis store -services have a max file upload of between 256Mb-512Mb. - -**Example JSON5 config:** -```json5 -"redis_store": { - "addresses": [ - "redis://127.0.0.1:6379/", - ], - "max_client_permits": 1000, -} -``` - -**Type:** [RedisSpec](#redisspec) - -### `noop` - -Noop store is a store that sends streams into the void and all data -retrieval will return 404 (`NotFound`). This can be useful for cases -where you may need to partition your data and part of your data needs -to be discarded. - -**Example JSON5 config:** -```json5 -"noop": {} -``` - -**Type:** [NoopSpec](#noopspec) - -### `experimental_mongo` - -Experimental `MongoDB` store implementation. - -This store uses `MongoDB` as a backend for storing data. It supports -both CAS (Content Addressable Storage) and scheduler data with -optional change streams for real-time updates. - -**Example JSON5 config:** -```json5 -"experimental_mongo": { - "connection_string": "mongodb://localhost:27017", - "database": "nativelink", - "cas_collection": "cas", - "key_prefix": "cas:", - "read_chunk_size": 65536, - "max_concurrent_uploads": 10, - "enable_change_streams": false, - "max_requests": "100" -} -``` - -**Type:** [ExperimentalMongoSpec](#experimentalmongospec) - ## CompressionAlgorithm Plus exactly one of the following variants (the key selects the variant): @@ -1574,7 +1017,7 @@ Configuration for an individual shard of the store. | Field | Type | Required | Default | Description | | --- | --- | --- | --- | --- | -| `store` | [StoreSpec](#storespec) | Yes | — | Store to shard the data to. | +| `store` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | Store to shard the data to. | | `weight` | integer (uint32) | — | 1 | The weight of the store. This is used to determine how much data should be sent to the store. The actual percentage is the sum of all the store's weights divided by the individual store's weight. | ## GrpcEndpoint diff --git a/web/apps/docs/content/docs/reference/nativelink-config/v1.3.0.mdx b/web/apps/docs/content/docs/reference/nativelink-config/v1.3.0.mdx index 7cb96983f..26775eeca 100644 --- a/web/apps/docs/content/docs/reference/nativelink-config/v1.3.0.mdx +++ b/web/apps/docs/content/docs/reference/nativelink-config/v1.3.0.mdx @@ -677,7 +677,7 @@ See [`experimental_cloud_object_store`](#experimental_cloud_object_store-1) for | Field | Type | Required | Default | Description | | --- | --- | --- | --- | --- | -| `backend` | [StoreSpec](#storespec) | Yes | — | The underlying store wrap around. All content will first flow through self before forwarding to backend. In the event there is an error detected in self, the connection to the backend will be terminated, and early termination should always cause updates to fail on the backend. | +| `backend` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | The underlying store wrap around. All content will first flow through self before forwarding to backend. In the event there is an error detected in self, the connection to the backend will be terminated, and early termination should always cause updates to fail on the backend. | | `verify_size` | boolean | — | `false` | If set the store will verify the size of the data before accepting an upload of data. | | `verify_hash` | boolean | — | `false` | If the data should be hashed and verify that the key matches the computed hash. The hash function is automatically determined based request and if not set will use the global default. | @@ -685,22 +685,22 @@ See [`experimental_cloud_object_store`](#experimental_cloud_object_store-1) for | Field | Type | Required | Default | Description | | --- | --- | --- | --- | --- | -| `backend` | [StoreSpec](#storespec) | Yes | — | The underlying store that will have it's results validated before sending to client. | -| `cas_store` | [StoreSpec](#storespec) | Yes | — | When a request is made, the results are decoded and all output digests/files are verified to exist in this CAS store before returning success. | +| `backend` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | The underlying store that will have it's results validated before sending to client. | +| `cas_store` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | When a request is made, the results are decoded and all output digests/files are verified to exist in this CAS store before returning success. | ## CompressionSpec | Field | Type | Required | Default | Description | | --- | --- | --- | --- | --- | -| `backend` | [StoreSpec](#storespec) | Yes | — | The underlying store wrap around. All content will first flow through self before forwarding to backend. In the event there is an error detected in self, the connection to the backend will be terminated, and early termination should always cause updates to fail on the backend. | +| `backend` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | The underlying store wrap around. All content will first flow through self before forwarding to backend. In the event there is an error detected in self, the connection to the backend will be terminated, and early termination should always cause updates to fail on the backend. | | `compression_algorithm` | [CompressionAlgorithm](#compressionalgorithm) | Yes | — | The compression algorithm to use. | ## DedupSpec | Field | Type | Required | Default | Description | | --- | --- | --- | --- | --- | -| `index_store` | [StoreSpec](#storespec) | Yes | — | Store used to store the index of each dedup slice. This store should generally be fast and small. | -| `content_store` | [StoreSpec](#storespec) | Yes | — | The store where the individual chunks will be uploaded. This store should generally be the slower & larger store. | +| `index_store` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | Store used to store the index of each dedup slice. This store should generally be fast and small. | +| `content_store` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | The store where the individual chunks will be uploaded. This store should generally be the slower & larger store. | | `min_size` | integer (uint32) | — | 65536 (64k) | Minimum size that a chunk will be when slicing up the content. Note: This setting can be increased to improve performance because it will actually not check this number of bytes when deciding where to partition the data. | | `normal_size` | integer (uint32) | — | 262144 (256k) | A best-effort attempt will be made to keep the average size of the chunks to this number. It is not a guarantee, but a slight attempt will be made. | | `max_size` | integer (uint32) | — | 524288 (512k) | Maximum size a chunk is allowed to be. | @@ -710,16 +710,16 @@ See [`experimental_cloud_object_store`](#experimental_cloud_object_store-1) for | Field | Type | Required | Default | Description | | --- | --- | --- | --- | --- | -| `backend` | [StoreSpec](#storespec) | Yes | — | The underlying store wrap around. All content will first flow through self before forwarding to backend. In the event there is an error detected in self, the connection to the backend will be terminated, and early termination should always cause updates to fail on the backend. | +| `backend` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | The underlying store wrap around. All content will first flow through self before forwarding to backend. In the event there is an error detected in self, the connection to the backend will be terminated, and early termination should always cause updates to fail on the backend. | | `eviction_policy` | [EvictionPolicy](#evictionpolicy) | — | — | Policy used to evict items out of the store. Failure to set this value will cause items to never be removed from the store causing infinite memory usage. | ## FastSlowSpec | Field | Type | Required | Default | Description | | --- | --- | --- | --- | --- | -| `fast` | [StoreSpec](#storespec) | Yes | — | Fast store that will be attempted to be contacted before reaching out to the `slow` store. | +| `fast` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | Fast store that will be attempted to be contacted before reaching out to the `slow` store. | | `fast_direction` | [StoreDirection](#storedirection) | — | `"both"` | How to handle the fast store. This can be useful to set to Get for worker nodes such that results are persisted to the slow store only. | -| `slow` | [StoreSpec](#storespec) | Yes | — | If the object does not exist in the `fast` store it will try to get it from this store. | +| `slow` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | If the object does not exist in the `fast` store it will try to get it from this store. | | `slow_direction` | [StoreDirection](#storedirection) | — | `"both"` | How to handle the slow store. This can be useful if creating a diode and you wish to have an upstream read only store. | ## ShardSpec @@ -750,8 +750,8 @@ See [`experimental_cloud_object_store`](#experimental_cloud_object_store-1) for | Field | Type | Required | Default | Description | | --- | --- | --- | --- | --- | | `size` | integer (uint64) | Yes | — | Size to partition the data on. | -| `lower_store` | [StoreSpec](#storespec) | Yes | — | Store to send data when object is < (less than) size. | -| `upper_store` | [StoreSpec](#storespec) | Yes | — | Store to send data when object is >= (less than eq) size. | +| `lower_store` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | Store to send data when object is < (less than) size. | +| `upper_store` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | Store to send data when object is >= (less than eq) size. | ## GrpcSpec @@ -987,563 +987,6 @@ The total delay is additive, so this example produces 9.525 to 15.875 s of total | `insecure_allow_http` | boolean | — | false | Allow unencrypted HTTP connections. Only use this for local testing. | | `disable_http2` | boolean | — | false | Disable HTTP/2 connections and only use HTTP/1.1. Default client configuration will have HTTP/1.1 and HTTP/2 enabled for connection schemes. HTTP/2 should be disabled if environments have poor support or performance related to HTTP/2. Safe to keep default unless underlying network environment, S3, or GCS API servers specify otherwise. | -## StoreSpec - -Plus exactly one of the following variants (the key selects the variant): - -### `memory` - -Memory store will store all data in a hash map in memory. - -**Example JSON5 config:** -```json5 -"memory": { - "eviction_policy": { - "max_bytes": "10mb", - } -} -``` - -**Type:** [MemorySpec](#memoryspec) - -### `experimental_cloud_object_store` - -A generic blob store that will store files on the cloud -provider. This configuration will never delete files, so you are -responsible for purging old files in other ways. -It supports the following backends: - -1. **Amazon S3:** - S3 store will use Amazon's S3 service as a backend to store - the files. This configuration can be used to share files - across multiple instances. Uses system certificates for TLS - verification via `rustls-platform-verifier`. - - **Example JSON5 config:** - ```json5 - "experimental_cloud_object_store": { - "provider": "aws", - "region": "eu-north-1", - "bucket": "crossplane-bucket-af79aeca9", - "key_prefix": "test-prefix-index/", - "retry": { - "max_retries": 6, - "delay": 0.3, - "jitter": 0.5 - }, - "multipart_max_concurrent_uploads": 10 - } - ``` - -2. **Google Cloud Storage:** - GCS store uses Google's GCS service as a backend to store - the files. This configuration can be used to share files - across multiple instances. - - **Example JSON5 config:** - ```json5 - "experimental_cloud_object_store": { - "provider": "gcs", - "bucket": "test-bucket", - "key_prefix": "test-prefix-index/", - "retry": { - "max_retries": 6, - "delay": 0.3, - "jitter": 0.5 - }, - "multipart_max_concurrent_uploads": 10 - } - ``` - -3. **Azure Blob Store:** - Azure Blob store will use Microsoft's Azure Blob service as a - backend to store the files. This configuration can be used to - share files across multiple instances. - - **Example JSON5 config:** - ```json5 - "experimental_cloud_object_store": { - "provider": "azure", - "account_name": "cloudshell1393657559", - "container": "simple-test-container", - "key_prefix": "folder/", - "retry": { - "max_retries": 6, - "delay": 0.3, - "jitter": 0.5 - }, - "multipart_max_concurrent_uploads": 10 - } - ``` - -4. **`NetApp` ONTAP S3** - `NetApp` ONTAP S3 store will use ONTAP's S3-compatible storage as a backend - to store files. This store is specifically configured for ONTAP's S3 requirements - including custom TLS configuration, credentials management, and proper vserver - configuration. - - This store uses AWS environment variables for credentials: - - `AWS_ACCESS_KEY_ID` - - `AWS_SECRET_ACCESS_KEY` - - `AWS_DEFAULT_REGION` - - **Example JSON5 config:** - ```json5 - "experimental_cloud_object_store": { - "provider": "ontap", - "endpoint": "https://ontap-s3-endpoint:443", - "vserver_name": "your-vserver", - "bucket": "your-bucket", - "root_certificates": "/path/to/certs.pem", // Optional - "key_prefix": "test-prefix/", // Optional - "retry": { - "max_retries": 6, - "delay": 0.3, - "jitter": 0.5 - }, - "multipart_max_concurrent_uploads": 10 - } - ``` - -**Type:** [ExperimentalCloudObjectSpec](#experimentalcloudobjectspec) - -### `ontap_s3_existence_cache` - -ONTAP S3 Existence Cache provides a caching layer on top of the ONTAP S3 store -to optimize repeated existence checks. It maintains an in-memory cache of object -digests and periodically syncs this cache to disk for persistence. - -The cache helps reduce latency for repeated calls to check object existence, -while still ensuring eventual consistency with the underlying ONTAP S3 store. - -Example JSON5 config: -```json5 -"ontap_s3_existence_cache": { - "index_path": "/path/to/cache/index.json", - "sync_interval_seconds": 300, - "backend": { - "endpoint": "https://ontap-s3-endpoint:443", - "vserver_name": "your-vserver", - "bucket": "your-bucket", - "key_prefix": "test-prefix/" - } -} -``` - -**Type:** [OntapS3ExistenceCacheSpec](#ontaps3existencecachespec) - -### `verify` - -Verify store is used to apply verifications to an underlying -store implementation. It is strongly encouraged to validate -as much data as you can before accepting data from a client, -failing to do so may cause the data in the store to be -populated with invalid data causing all kinds of problems. - -The suggested configuration is to have the CAS validate the -hash and size and the AC validate nothing. - -**Example JSON5 config:** -```json5 -"verify": { - "backend": { - "memory": { - "eviction_policy": { - "max_bytes": "500mb" - } - }, - }, - "verify_size": true, - "verify_hash": true -} -``` - -**Type:** [VerifySpec](#verifyspec) - -### `completeness_checking` - -Completeness checking store verifies if the -output files & folders exist in the CAS before forwarding -the request to the underlying store. -Note: This store should only be used on AC stores. - -**Example JSON5 config:** -```json5 -"completeness_checking": { - "backend": { - "filesystem": { - "content_path": "~/.cache/nativelink/content_path-ac", - "temp_path": "~/.cache/nativelink/tmp_path-ac", - "eviction_policy": { - "max_bytes": "500mb", - } - } - }, - "cas_store": { - "ref_store": { - "name": "CAS_MAIN_STORE" - } - } -} -``` - -**Type:** [CompletenessCheckingSpec](#completenesscheckingspec) - -### `compression` - -A compression store that will compress the data inbound and -outbound. There will be a non-trivial cost to compress and -decompress the data, but in many cases if the final store is -a store that requires network transport and/or storage space -is a concern it is often faster and more efficient to use this -store before those stores. - -**Example JSON5 config:** -```json5 -"compression": { - "compression_algorithm": { - "lz4": {} - }, - "backend": { - "filesystem": { - "content_path": "/tmp/nativelink/data/content_path-cas", - "temp_path": "/tmp/nativelink/data/tmp_path-cas", - "eviction_policy": { - "max_bytes": "2gb", - } - } - } -} -``` - -**Type:** [CompressionSpec](#compressionspec) - -### `dedup` - -A dedup store will take the inputs and run a rolling hash -algorithm on them to slice the input into smaller parts then -run a sha256 algorithm on the slice and if the object doesn't -already exist, upload the slice to the `content_store` using -a new digest of just the slice. Once all parts exist, an -Action-Cache-like digest will be built and uploaded to the -`index_store` which will contain a reference to each -chunk/digest of the uploaded file. Downloading a request will -first grab the index from the `index_store`, and forward the -download content of each chunk as if it were one file. - -This store is exceptionally good when the following conditions -are met: -* Content is mostly the same (inserts, updates, deletes are ok) -* Content is not compressed or encrypted -* Uploading or downloading from `content_store` is the bottleneck. - -Note: This store pairs well when used with `CompressionSpec` as -the `content_store`, but never put `DedupSpec` as the backend of -`CompressionSpec` as it will negate all the gains. - -Note: When running `.has()` on this store, it will only check -to see if the entry exists in the `index_store` and not check -if the individual chunks exist in the `content_store`. - -**Example JSON5 config:** -```json5 -"dedup": { - "index_store": { - "memory": { - "eviction_policy": { - "max_bytes": "1GB", - } - } - }, - "content_store": { - "compression": { - "compression_algorithm": { - "lz4": {} - }, - "backend": { - "fast_slow": { - "fast": { - "memory": { - "eviction_policy": { - "max_bytes": "500MB", - } - } - }, - "slow": { - "filesystem": { - "content_path": "/tmp/nativelink/data/content_path-content", - "temp_path": "/tmp/nativelink/data/tmp_path-content", - "eviction_policy": { - "max_bytes": "2gb" - } - } - } - } - } - } - } -} -``` - -**Type:** [DedupSpec](#dedupspec) - -### `existence_cache` - -Existence store will wrap around another store and cache calls -to has so that subsequent `has_with_results` calls will be -faster. This is useful for cases when you have a store that -is slow to respond to has calls. -Note: This store should only be used on CAS stores. - -**Example JSON5 config:** -```json5 -"existence_cache": { - "backend": { - "memory": { - "eviction_policy": { - "max_bytes": "500mb", - } - } - }, - // Note this is the existence store policy, not the backend policy - "eviction_policy": { - "max_seconds": 100, - } -} -``` - -**Type:** [ExistenceCacheSpec](#existencecachespec) - -### `fast_slow` - -`FastSlow` store will first try to fetch the data from the `fast` -store and then if it does not exist try the `slow` store. -When the object does exist in the `slow` store, it will copy -the data to the `fast` store while returning the data. -This store should be thought of as a store that "buffers" -the data to the `fast` store. -On uploads it will mirror data to both `fast` and `slow` stores. - -WARNING: If you need data to always exist in the `slow` store -for something like remote execution, be careful because this -store will never check to see if the objects exist in the -`slow` store if it exists in the `fast` store (i.e., it assumes -that if an object exists in the `fast` store it will exist in -the `slow` store). - -***Example JSON5 config:*** -```json5 -"fast_slow": { - "fast": { - "filesystem": { - "content_path": "/tmp/nativelink/data/content_path-index", - "temp_path": "/tmp/nativelink/data/tmp_path-index", - "eviction_policy": { - "max_bytes": "500mb", - } - } - }, - "slow": { - "filesystem": { - "content_path": "/tmp/nativelink/data/content_path-index", - "temp_path": "/tmp/nativelink/data/tmp_path-index", - "eviction_policy": { - "max_bytes": "500mb", - } - } - } -} -``` - -**Type:** [FastSlowSpec](#fastslowspec) - -### `shard` - -Shards the data to multiple stores. This is useful for cases -when you want to distribute the load across multiple stores. -The digest hash is used to determine which store to send the -data to. - -**Example JSON5 config:** -```json5 -"shard": { - "stores": [ - { - "store": { - "memory": { - "eviction_policy": { - "max_bytes": "10mb" - }, - }, - }, - "weight": 1 - }] -} -``` - -**Type:** [ShardSpec](#shardspec) - -### `filesystem` - -Stores the data on the filesystem. This store is designed for -local persistent storage. Restarts of this program should restore -the previous state, meaning anything uploaded will be persistent -as long as the filesystem integrity holds. - -**Example JSON5 config:** -```json5 -"filesystem": { - "content_path": "/tmp/nativelink/data-worker-test/content_path-cas", - "temp_path": "/tmp/nativelink/data-worker-test/tmp_path-cas", - "eviction_policy": { - "max_bytes": "10gb", - } -} -``` - -**Type:** [FilesystemSpec](#filesystemspec) - -### `ref_store` - -Store used to reference a store in the root store manager. -This is useful for cases when you want to share a store in different -nested stores. Example, you may want to share the same memory store -used for the action cache, but use a `FastSlowSpec` and have the fast -store also share the memory store for efficiency. - -**Example JSON5 config:** -```json5 -"ref_store": { - "name": "FS_CONTENT_STORE" -} -``` - -**Type:** [RefSpec](#refspec) - -### `size_partitioning` - -Uses the size field of the digest to separate which store to send the -data. This is useful for cases when you'd like to put small objects -in one store and large objects in another store. This should only be -used if the size field is the real size of the content, in other -words, don't use on AC (Action Cache) stores. Any store where you can -safely use `VerifySpec.verify_size = true`, this store should be safe -to use (i.e., CAS stores). - -**Example JSON5 config:** -```json5 -"size_partitioning": { - "size": "128mib", - "lower_store": { - "memory": { - "eviction_policy": { - "max_bytes": "${NATIVELINK_CAS_MEMORY_CONTENT_LIMIT:-100mb}" - } - } - }, - "upper_store": { - /// This store discards data larger than 128mib. - "noop": {} - } -} -``` - -**Type:** [SizePartitioningSpec](#sizepartitioningspec) - -### `grpc` - -This store will pass-through calls to another GRPC store. This store -is not designed to be used as a sub-store of another store, but it -does satisfy the interface and will likely work. - -One major GOTCHA is that some stores use a special function on this -store to get the size of the underlying object, which is only reliable -when this store is serving the a CAS store, not an AC store. If using -this store directly without being a child of any store there are no -side effects and is the most efficient way to use it. - -**Example JSON5 config:** -```json5 -"grpc": { - "instance_name": "main", - "endpoints": [ - {"address": "grpc://${CAS_ENDPOINT:-127.0.0.1}:50051"} - ], - "connections_per_endpoint": "5", - "rpc_timeout_s": "5m", - "store_type": "ac", - // Static headers attached to every outgoing request to the upstream - // remote cache. Useful for fixed service-account credentials. - "headers": { - "authorization": "Bearer my-static-token" - }, - // Header names to copy from the inbound client request and forward to - // the upstream remote cache. Use this to pass through dynamic - // credentials such as a JWT sent by the build client. - "forward_headers": ["authorization", "x-custom-token"] -} -``` - -**Type:** [GrpcSpec](#grpcspec) - -### `redis_store` - -Stores data in any stores compatible with Redis APIs. - -Pairs well with `SizePartitioning` and/or `FastSlow` stores. -Ideal for accepting small object sizes as most Redis store -services have a max file upload of between 256Mb-512Mb. - -**Example JSON5 config:** -```json5 -"redis_store": { - "addresses": [ - "redis://127.0.0.1:6379/", - ], - "max_client_permits": 1000, -} -``` - -**Type:** [RedisSpec](#redisspec) - -### `noop` - -Noop store is a store that sends streams into the void and all data -retrieval will return 404 (`NotFound`). This can be useful for cases -where you may need to partition your data and part of your data needs -to be discarded. - -**Example JSON5 config:** -```json5 -"noop": {} -``` - -**Type:** [NoopSpec](#noopspec) - -### `experimental_mongo` - -Experimental `MongoDB` store implementation. - -This store uses `MongoDB` as a backend for storing data. It supports -both CAS (Content Addressable Storage) and scheduler data with -optional change streams for real-time updates. - -**Example JSON5 config:** -```json5 -"experimental_mongo": { - "connection_string": "mongodb://localhost:27017", - "database": "nativelink", - "cas_collection": "cas", - "key_prefix": "cas:", - "read_chunk_size": 65536, - "max_concurrent_uploads": 10, - "enable_change_streams": false, - "max_requests": "100" -} -``` - -**Type:** [ExperimentalMongoSpec](#experimentalmongospec) - ## CompressionAlgorithm Plus exactly one of the following variants (the key selects the variant): @@ -1575,7 +1018,7 @@ Configuration for an individual shard of the store. | Field | Type | Required | Default | Description | | --- | --- | --- | --- | --- | -| `store` | [StoreSpec](#storespec) | Yes | — | Store to shard the data to. | +| `store` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | Store to shard the data to. | | `weight` | integer (uint32) | — | 1 | The weight of the store. This is used to determine how much data should be sent to the store. The actual percentage is the sum of all the store's weights divided by the individual store's weight. | ## GrpcEndpoint diff --git a/web/apps/docs/content/docs/reference/nativelink-config/v1.3.1.mdx b/web/apps/docs/content/docs/reference/nativelink-config/v1.3.1.mdx index 24c1f5891..b582228af 100644 --- a/web/apps/docs/content/docs/reference/nativelink-config/v1.3.1.mdx +++ b/web/apps/docs/content/docs/reference/nativelink-config/v1.3.1.mdx @@ -677,7 +677,7 @@ See [`experimental_cloud_object_store`](#experimental_cloud_object_store-1) for | Field | Type | Required | Default | Description | | --- | --- | --- | --- | --- | -| `backend` | [StoreSpec](#storespec) | Yes | — | The underlying store wrap around. All content will first flow through self before forwarding to backend. In the event there is an error detected in self, the connection to the backend will be terminated, and early termination should always cause updates to fail on the backend. | +| `backend` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | The underlying store wrap around. All content will first flow through self before forwarding to backend. In the event there is an error detected in self, the connection to the backend will be terminated, and early termination should always cause updates to fail on the backend. | | `verify_size` | boolean | — | `false` | If set the store will verify the size of the data before accepting an upload of data. | | `verify_hash` | boolean | — | `false` | If the data should be hashed and verify that the key matches the computed hash. The hash function is automatically determined based request and if not set will use the global default. | @@ -685,22 +685,22 @@ See [`experimental_cloud_object_store`](#experimental_cloud_object_store-1) for | Field | Type | Required | Default | Description | | --- | --- | --- | --- | --- | -| `backend` | [StoreSpec](#storespec) | Yes | — | The underlying store that will have it's results validated before sending to client. | -| `cas_store` | [StoreSpec](#storespec) | Yes | — | When a request is made, the results are decoded and all output digests/files are verified to exist in this CAS store before returning success. | +| `backend` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | The underlying store that will have it's results validated before sending to client. | +| `cas_store` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | When a request is made, the results are decoded and all output digests/files are verified to exist in this CAS store before returning success. | ## CompressionSpec | Field | Type | Required | Default | Description | | --- | --- | --- | --- | --- | -| `backend` | [StoreSpec](#storespec) | Yes | — | The underlying store wrap around. All content will first flow through self before forwarding to backend. In the event there is an error detected in self, the connection to the backend will be terminated, and early termination should always cause updates to fail on the backend. | +| `backend` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | The underlying store wrap around. All content will first flow through self before forwarding to backend. In the event there is an error detected in self, the connection to the backend will be terminated, and early termination should always cause updates to fail on the backend. | | `compression_algorithm` | [CompressionAlgorithm](#compressionalgorithm) | Yes | — | The compression algorithm to use. | ## DedupSpec | Field | Type | Required | Default | Description | | --- | --- | --- | --- | --- | -| `index_store` | [StoreSpec](#storespec) | Yes | — | Store used to store the index of each dedup slice. This store should generally be fast and small. | -| `content_store` | [StoreSpec](#storespec) | Yes | — | The store where the individual chunks will be uploaded. This store should generally be the slower & larger store. | +| `index_store` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | Store used to store the index of each dedup slice. This store should generally be fast and small. | +| `content_store` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | The store where the individual chunks will be uploaded. This store should generally be the slower & larger store. | | `min_size` | integer (uint32) | — | 65536 (64k) | Minimum size that a chunk will be when slicing up the content. Note: This setting can be increased to improve performance because it will actually not check this number of bytes when deciding where to partition the data. | | `normal_size` | integer (uint32) | — | 262144 (256k) | A best-effort attempt will be made to keep the average size of the chunks to this number. It is not a guarantee, but a slight attempt will be made. | | `max_size` | integer (uint32) | — | 524288 (512k) | Maximum size a chunk is allowed to be. | @@ -710,16 +710,16 @@ See [`experimental_cloud_object_store`](#experimental_cloud_object_store-1) for | Field | Type | Required | Default | Description | | --- | --- | --- | --- | --- | -| `backend` | [StoreSpec](#storespec) | Yes | — | The underlying store wrap around. All content will first flow through self before forwarding to backend. In the event there is an error detected in self, the connection to the backend will be terminated, and early termination should always cause updates to fail on the backend. | +| `backend` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | The underlying store wrap around. All content will first flow through self before forwarding to backend. In the event there is an error detected in self, the connection to the backend will be terminated, and early termination should always cause updates to fail on the backend. | | `eviction_policy` | [EvictionPolicy](#evictionpolicy) | — | — | Policy used to evict items out of the store. Failure to set this value will cause items to never be removed from the store causing infinite memory usage. | ## FastSlowSpec | Field | Type | Required | Default | Description | | --- | --- | --- | --- | --- | -| `fast` | [StoreSpec](#storespec) | Yes | — | Fast store that will be attempted to be contacted before reaching out to the `slow` store. | +| `fast` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | Fast store that will be attempted to be contacted before reaching out to the `slow` store. | | `fast_direction` | [StoreDirection](#storedirection) | — | `"both"` | How to handle the fast store. This can be useful to set to Get for worker nodes such that results are persisted to the slow store only. | -| `slow` | [StoreSpec](#storespec) | Yes | — | If the object does not exist in the `fast` store it will try to get it from this store. | +| `slow` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | If the object does not exist in the `fast` store it will try to get it from this store. | | `slow_direction` | [StoreDirection](#storedirection) | — | `"both"` | How to handle the slow store. This can be useful if creating a diode and you wish to have an upstream read only store. | ## ShardSpec @@ -750,8 +750,8 @@ See [`experimental_cloud_object_store`](#experimental_cloud_object_store-1) for | Field | Type | Required | Default | Description | | --- | --- | --- | --- | --- | | `size` | integer (uint64) | Yes | — | Size to partition the data on. | -| `lower_store` | [StoreSpec](#storespec) | Yes | — | Store to send data when object is < (less than) size. | -| `upper_store` | [StoreSpec](#storespec) | Yes | — | Store to send data when object is >= (less than eq) size. | +| `lower_store` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | Store to send data when object is < (less than) size. | +| `upper_store` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | Store to send data when object is >= (less than eq) size. | ## GrpcSpec @@ -987,563 +987,6 @@ The total delay is additive, so this example produces 9.525 to 15.875 s of total | `insecure_allow_http` | boolean | — | false | Allow unencrypted HTTP connections. Only use this for local testing. | | `disable_http2` | boolean | — | false | Disable HTTP/2 connections and only use HTTP/1.1. Default client configuration will have HTTP/1.1 and HTTP/2 enabled for connection schemes. HTTP/2 should be disabled if environments have poor support or performance related to HTTP/2. Safe to keep default unless underlying network environment, S3, or GCS API servers specify otherwise. | -## StoreSpec - -Plus exactly one of the following variants (the key selects the variant): - -### `memory` - -Memory store will store all data in a hash map in memory. - -**Example JSON5 config:** -```json5 -"memory": { - "eviction_policy": { - "max_bytes": "10mb", - } -} -``` - -**Type:** [MemorySpec](#memoryspec) - -### `experimental_cloud_object_store` - -A generic blob store that will store files on the cloud -provider. This configuration will never delete files, so you are -responsible for purging old files in other ways. -It supports the following backends: - -1. **Amazon S3:** - S3 store will use Amazon's S3 service as a backend to store - the files. This configuration can be used to share files - across multiple instances. Uses system certificates for TLS - verification via `rustls-platform-verifier`. - - **Example JSON5 config:** - ```json5 - "experimental_cloud_object_store": { - "provider": "aws", - "region": "eu-north-1", - "bucket": "crossplane-bucket-af79aeca9", - "key_prefix": "test-prefix-index/", - "retry": { - "max_retries": 6, - "delay": 0.3, - "jitter": 0.5 - }, - "multipart_max_concurrent_uploads": 10 - } - ``` - -2. **Google Cloud Storage:** - GCS store uses Google's GCS service as a backend to store - the files. This configuration can be used to share files - across multiple instances. - - **Example JSON5 config:** - ```json5 - "experimental_cloud_object_store": { - "provider": "gcs", - "bucket": "test-bucket", - "key_prefix": "test-prefix-index/", - "retry": { - "max_retries": 6, - "delay": 0.3, - "jitter": 0.5 - }, - "multipart_max_concurrent_uploads": 10 - } - ``` - -3. **Azure Blob Store:** - Azure Blob store will use Microsoft's Azure Blob service as a - backend to store the files. This configuration can be used to - share files across multiple instances. - - **Example JSON5 config:** - ```json5 - "experimental_cloud_object_store": { - "provider": "azure", - "account_name": "cloudshell1393657559", - "container": "simple-test-container", - "key_prefix": "folder/", - "retry": { - "max_retries": 6, - "delay": 0.3, - "jitter": 0.5 - }, - "multipart_max_concurrent_uploads": 10 - } - ``` - -4. **`NetApp` ONTAP S3** - `NetApp` ONTAP S3 store will use ONTAP's S3-compatible storage as a backend - to store files. This store is specifically configured for ONTAP's S3 requirements - including custom TLS configuration, credentials management, and proper vserver - configuration. - - This store uses AWS environment variables for credentials: - - `AWS_ACCESS_KEY_ID` - - `AWS_SECRET_ACCESS_KEY` - - `AWS_DEFAULT_REGION` - - **Example JSON5 config:** - ```json5 - "experimental_cloud_object_store": { - "provider": "ontap", - "endpoint": "https://ontap-s3-endpoint:443", - "vserver_name": "your-vserver", - "bucket": "your-bucket", - "root_certificates": "/path/to/certs.pem", // Optional - "key_prefix": "test-prefix/", // Optional - "retry": { - "max_retries": 6, - "delay": 0.3, - "jitter": 0.5 - }, - "multipart_max_concurrent_uploads": 10 - } - ``` - -**Type:** [ExperimentalCloudObjectSpec](#experimentalcloudobjectspec) - -### `ontap_s3_existence_cache` - -ONTAP S3 Existence Cache provides a caching layer on top of the ONTAP S3 store -to optimize repeated existence checks. It maintains an in-memory cache of object -digests and periodically syncs this cache to disk for persistence. - -The cache helps reduce latency for repeated calls to check object existence, -while still ensuring eventual consistency with the underlying ONTAP S3 store. - -Example JSON5 config: -```json5 -"ontap_s3_existence_cache": { - "index_path": "/path/to/cache/index.json", - "sync_interval_seconds": 300, - "backend": { - "endpoint": "https://ontap-s3-endpoint:443", - "vserver_name": "your-vserver", - "bucket": "your-bucket", - "key_prefix": "test-prefix/" - } -} -``` - -**Type:** [OntapS3ExistenceCacheSpec](#ontaps3existencecachespec) - -### `verify` - -Verify store is used to apply verifications to an underlying -store implementation. It is strongly encouraged to validate -as much data as you can before accepting data from a client, -failing to do so may cause the data in the store to be -populated with invalid data causing all kinds of problems. - -The suggested configuration is to have the CAS validate the -hash and size and the AC validate nothing. - -**Example JSON5 config:** -```json5 -"verify": { - "backend": { - "memory": { - "eviction_policy": { - "max_bytes": "500mb" - } - }, - }, - "verify_size": true, - "verify_hash": true -} -``` - -**Type:** [VerifySpec](#verifyspec) - -### `completeness_checking` - -Completeness checking store verifies if the -output files & folders exist in the CAS before forwarding -the request to the underlying store. -Note: This store should only be used on AC stores. - -**Example JSON5 config:** -```json5 -"completeness_checking": { - "backend": { - "filesystem": { - "content_path": "~/.cache/nativelink/content_path-ac", - "temp_path": "~/.cache/nativelink/tmp_path-ac", - "eviction_policy": { - "max_bytes": "500mb", - } - } - }, - "cas_store": { - "ref_store": { - "name": "CAS_MAIN_STORE" - } - } -} -``` - -**Type:** [CompletenessCheckingSpec](#completenesscheckingspec) - -### `compression` - -A compression store that will compress the data inbound and -outbound. There will be a non-trivial cost to compress and -decompress the data, but in many cases if the final store is -a store that requires network transport and/or storage space -is a concern it is often faster and more efficient to use this -store before those stores. - -**Example JSON5 config:** -```json5 -"compression": { - "compression_algorithm": { - "lz4": {} - }, - "backend": { - "filesystem": { - "content_path": "/tmp/nativelink/data/content_path-cas", - "temp_path": "/tmp/nativelink/data/tmp_path-cas", - "eviction_policy": { - "max_bytes": "2gb", - } - } - } -} -``` - -**Type:** [CompressionSpec](#compressionspec) - -### `dedup` - -A dedup store will take the inputs and run a rolling hash -algorithm on them to slice the input into smaller parts then -run a sha256 algorithm on the slice and if the object doesn't -already exist, upload the slice to the `content_store` using -a new digest of just the slice. Once all parts exist, an -Action-Cache-like digest will be built and uploaded to the -`index_store` which will contain a reference to each -chunk/digest of the uploaded file. Downloading a request will -first grab the index from the `index_store`, and forward the -download content of each chunk as if it were one file. - -This store is exceptionally good when the following conditions -are met: -* Content is mostly the same (inserts, updates, deletes are ok) -* Content is not compressed or encrypted -* Uploading or downloading from `content_store` is the bottleneck. - -Note: This store pairs well when used with `CompressionSpec` as -the `content_store`, but never put `DedupSpec` as the backend of -`CompressionSpec` as it will negate all the gains. - -Note: When running `.has()` on this store, it will only check -to see if the entry exists in the `index_store` and not check -if the individual chunks exist in the `content_store`. - -**Example JSON5 config:** -```json5 -"dedup": { - "index_store": { - "memory": { - "eviction_policy": { - "max_bytes": "1GB", - } - } - }, - "content_store": { - "compression": { - "compression_algorithm": { - "lz4": {} - }, - "backend": { - "fast_slow": { - "fast": { - "memory": { - "eviction_policy": { - "max_bytes": "500MB", - } - } - }, - "slow": { - "filesystem": { - "content_path": "/tmp/nativelink/data/content_path-content", - "temp_path": "/tmp/nativelink/data/tmp_path-content", - "eviction_policy": { - "max_bytes": "2gb" - } - } - } - } - } - } - } -} -``` - -**Type:** [DedupSpec](#dedupspec) - -### `existence_cache` - -Existence store will wrap around another store and cache calls -to has so that subsequent `has_with_results` calls will be -faster. This is useful for cases when you have a store that -is slow to respond to has calls. -Note: This store should only be used on CAS stores. - -**Example JSON5 config:** -```json5 -"existence_cache": { - "backend": { - "memory": { - "eviction_policy": { - "max_bytes": "500mb", - } - } - }, - // Note this is the existence store policy, not the backend policy - "eviction_policy": { - "max_seconds": 100, - } -} -``` - -**Type:** [ExistenceCacheSpec](#existencecachespec) - -### `fast_slow` - -`FastSlow` store will first try to fetch the data from the `fast` -store and then if it does not exist try the `slow` store. -When the object does exist in the `slow` store, it will copy -the data to the `fast` store while returning the data. -This store should be thought of as a store that "buffers" -the data to the `fast` store. -On uploads it will mirror data to both `fast` and `slow` stores. - -WARNING: If you need data to always exist in the `slow` store -for something like remote execution, be careful because this -store will never check to see if the objects exist in the -`slow` store if it exists in the `fast` store (i.e., it assumes -that if an object exists in the `fast` store it will exist in -the `slow` store). - -***Example JSON5 config:*** -```json5 -"fast_slow": { - "fast": { - "filesystem": { - "content_path": "/tmp/nativelink/data/content_path-index", - "temp_path": "/tmp/nativelink/data/tmp_path-index", - "eviction_policy": { - "max_bytes": "500mb", - } - } - }, - "slow": { - "filesystem": { - "content_path": "/tmp/nativelink/data/content_path-index", - "temp_path": "/tmp/nativelink/data/tmp_path-index", - "eviction_policy": { - "max_bytes": "500mb", - } - } - } -} -``` - -**Type:** [FastSlowSpec](#fastslowspec) - -### `shard` - -Shards the data to multiple stores. This is useful for cases -when you want to distribute the load across multiple stores. -The digest hash is used to determine which store to send the -data to. - -**Example JSON5 config:** -```json5 -"shard": { - "stores": [ - { - "store": { - "memory": { - "eviction_policy": { - "max_bytes": "10mb" - }, - }, - }, - "weight": 1 - }] -} -``` - -**Type:** [ShardSpec](#shardspec) - -### `filesystem` - -Stores the data on the filesystem. This store is designed for -local persistent storage. Restarts of this program should restore -the previous state, meaning anything uploaded will be persistent -as long as the filesystem integrity holds. - -**Example JSON5 config:** -```json5 -"filesystem": { - "content_path": "/tmp/nativelink/data-worker-test/content_path-cas", - "temp_path": "/tmp/nativelink/data-worker-test/tmp_path-cas", - "eviction_policy": { - "max_bytes": "10gb", - } -} -``` - -**Type:** [FilesystemSpec](#filesystemspec) - -### `ref_store` - -Store used to reference a store in the root store manager. -This is useful for cases when you want to share a store in different -nested stores. Example, you may want to share the same memory store -used for the action cache, but use a `FastSlowSpec` and have the fast -store also share the memory store for efficiency. - -**Example JSON5 config:** -```json5 -"ref_store": { - "name": "FS_CONTENT_STORE" -} -``` - -**Type:** [RefSpec](#refspec) - -### `size_partitioning` - -Uses the size field of the digest to separate which store to send the -data. This is useful for cases when you'd like to put small objects -in one store and large objects in another store. This should only be -used if the size field is the real size of the content, in other -words, don't use on AC (Action Cache) stores. Any store where you can -safely use `VerifySpec.verify_size = true`, this store should be safe -to use (i.e., CAS stores). - -**Example JSON5 config:** -```json5 -"size_partitioning": { - "size": "128mib", - "lower_store": { - "memory": { - "eviction_policy": { - "max_bytes": "${NATIVELINK_CAS_MEMORY_CONTENT_LIMIT:-100mb}" - } - } - }, - "upper_store": { - /// This store discards data larger than 128mib. - "noop": {} - } -} -``` - -**Type:** [SizePartitioningSpec](#sizepartitioningspec) - -### `grpc` - -This store will pass-through calls to another GRPC store. This store -is not designed to be used as a sub-store of another store, but it -does satisfy the interface and will likely work. - -One major GOTCHA is that some stores use a special function on this -store to get the size of the underlying object, which is only reliable -when this store is serving the a CAS store, not an AC store. If using -this store directly without being a child of any store there are no -side effects and is the most efficient way to use it. - -**Example JSON5 config:** -```json5 -"grpc": { - "instance_name": "main", - "endpoints": [ - {"address": "grpc://${CAS_ENDPOINT:-127.0.0.1}:50051"} - ], - "connections_per_endpoint": "5", - "rpc_timeout_s": "5m", - "store_type": "ac", - // Static headers attached to every outgoing request to the upstream - // remote cache. Useful for fixed service-account credentials. - "headers": { - "authorization": "Bearer my-static-token" - }, - // Header names to copy from the inbound client request and forward to - // the upstream remote cache. Use this to pass through dynamic - // credentials such as a JWT sent by the build client. - "forward_headers": ["authorization", "x-custom-token"] -} -``` - -**Type:** [GrpcSpec](#grpcspec) - -### `redis_store` - -Stores data in any stores compatible with Redis APIs. - -Pairs well with `SizePartitioning` and/or `FastSlow` stores. -Ideal for accepting small object sizes as most Redis store -services have a max file upload of between 256Mb-512Mb. - -**Example JSON5 config:** -```json5 -"redis_store": { - "addresses": [ - "redis://127.0.0.1:6379/", - ], - "max_client_permits": 1000, -} -``` - -**Type:** [RedisSpec](#redisspec) - -### `noop` - -Noop store is a store that sends streams into the void and all data -retrieval will return 404 (`NotFound`). This can be useful for cases -where you may need to partition your data and part of your data needs -to be discarded. - -**Example JSON5 config:** -```json5 -"noop": {} -``` - -**Type:** [NoopSpec](#noopspec) - -### `experimental_mongo` - -Experimental `MongoDB` store implementation. - -This store uses `MongoDB` as a backend for storing data. It supports -both CAS (Content Addressable Storage) and scheduler data with -optional change streams for real-time updates. - -**Example JSON5 config:** -```json5 -"experimental_mongo": { - "connection_string": "mongodb://localhost:27017", - "database": "nativelink", - "cas_collection": "cas", - "key_prefix": "cas:", - "read_chunk_size": 65536, - "max_concurrent_uploads": 10, - "enable_change_streams": false, - "max_requests": "100" -} -``` - -**Type:** [ExperimentalMongoSpec](#experimentalmongospec) - ## CompressionAlgorithm Plus exactly one of the following variants (the key selects the variant): @@ -1575,7 +1018,7 @@ Configuration for an individual shard of the store. | Field | Type | Required | Default | Description | | --- | --- | --- | --- | --- | -| `store` | [StoreSpec](#storespec) | Yes | — | Store to shard the data to. | +| `store` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | Store to shard the data to. | | `weight` | integer (uint32) | — | 1 | The weight of the store. This is used to determine how much data should be sent to the store. The actual percentage is the sum of all the store's weights divided by the individual store's weight. | ## GrpcEndpoint diff --git a/web/apps/docs/content/docs/reference/nativelink-config/v1.3.2.mdx b/web/apps/docs/content/docs/reference/nativelink-config/v1.3.2.mdx index 27bf375fa..4d91fd520 100644 --- a/web/apps/docs/content/docs/reference/nativelink-config/v1.3.2.mdx +++ b/web/apps/docs/content/docs/reference/nativelink-config/v1.3.2.mdx @@ -684,7 +684,7 @@ Plus exactly one of the following variants (the key selects the variant): | Field | Type | Required | Default | Description | | --- | --- | --- | --- | --- | | `cache_type` | string | Yes | — | Low-cardinality cache type label for metrics, for example `cas` or `ac`. | -| `backend` | [StoreSpec](#storespec) | Yes | — | Store to wrap with cache operation metrics. | +| `backend` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | Store to wrap with cache operation metrics. | ## MemorySpec @@ -708,7 +708,7 @@ See [`experimental_cloud_object_store`](#experimental_cloud_object_store-1) for | Field | Type | Required | Default | Description | | --- | --- | --- | --- | --- | -| `backend` | [StoreSpec](#storespec) | Yes | — | The underlying store wrap around. All content will first flow through self before forwarding to backend. In the event there is an error detected in self, the connection to the backend will be terminated, and early termination should always cause updates to fail on the backend. | +| `backend` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | The underlying store wrap around. All content will first flow through self before forwarding to backend. In the event there is an error detected in self, the connection to the backend will be terminated, and early termination should always cause updates to fail on the backend. | | `verify_size` | boolean | — | `false` | If set the store will verify the size of the data before accepting an upload of data. | | `verify_hash` | boolean | — | `false` | If the data should be hashed and verify that the key matches the computed hash. The hash function is automatically determined based request and if not set will use the global default. | @@ -716,22 +716,22 @@ See [`experimental_cloud_object_store`](#experimental_cloud_object_store-1) for | Field | Type | Required | Default | Description | | --- | --- | --- | --- | --- | -| `backend` | [StoreSpec](#storespec) | Yes | — | The underlying store that will have it's results validated before sending to client. | -| `cas_store` | [StoreSpec](#storespec) | Yes | — | When a request is made, the results are decoded and all output digests/files are verified to exist in this CAS store before returning success. | +| `backend` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | The underlying store that will have it's results validated before sending to client. | +| `cas_store` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | When a request is made, the results are decoded and all output digests/files are verified to exist in this CAS store before returning success. | ## CompressionSpec | Field | Type | Required | Default | Description | | --- | --- | --- | --- | --- | -| `backend` | [StoreSpec](#storespec) | Yes | — | The underlying store wrap around. All content will first flow through self before forwarding to backend. In the event there is an error detected in self, the connection to the backend will be terminated, and early termination should always cause updates to fail on the backend. | +| `backend` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | The underlying store wrap around. All content will first flow through self before forwarding to backend. In the event there is an error detected in self, the connection to the backend will be terminated, and early termination should always cause updates to fail on the backend. | | `compression_algorithm` | [CompressionAlgorithm](#compressionalgorithm) | Yes | — | The compression algorithm to use. | ## DedupSpec | Field | Type | Required | Default | Description | | --- | --- | --- | --- | --- | -| `index_store` | [StoreSpec](#storespec) | Yes | — | Store used to store the index of each dedup slice. This store should generally be fast and small. | -| `content_store` | [StoreSpec](#storespec) | Yes | — | The store where the individual chunks will be uploaded. This store should generally be the slower & larger store. | +| `index_store` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | Store used to store the index of each dedup slice. This store should generally be fast and small. | +| `content_store` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | The store where the individual chunks will be uploaded. This store should generally be the slower & larger store. | | `min_size` | integer (uint32) | — | 65536 (64k) | Minimum size that a chunk will be when slicing up the content. Note: This setting can be increased to improve performance because it will actually not check this number of bytes when deciding where to partition the data. | | `normal_size` | integer (uint32) | — | 262144 (256k) | A best-effort attempt will be made to keep the average size of the chunks to this number. It is not a guarantee, but a slight attempt will be made. | | `max_size` | integer (uint32) | — | 524288 (512k) | Maximum size a chunk is allowed to be. | @@ -741,16 +741,16 @@ See [`experimental_cloud_object_store`](#experimental_cloud_object_store-1) for | Field | Type | Required | Default | Description | | --- | --- | --- | --- | --- | -| `backend` | [StoreSpec](#storespec) | Yes | — | The underlying store wrap around. All content will first flow through self before forwarding to backend. In the event there is an error detected in self, the connection to the backend will be terminated, and early termination should always cause updates to fail on the backend. | +| `backend` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | The underlying store wrap around. All content will first flow through self before forwarding to backend. In the event there is an error detected in self, the connection to the backend will be terminated, and early termination should always cause updates to fail on the backend. | | `eviction_policy` | [EvictionPolicy](#evictionpolicy) | — | — | Policy used to evict items out of the store. Failure to set this value will cause items to never be removed from the store causing infinite memory usage. | ## FastSlowSpec | Field | Type | Required | Default | Description | | --- | --- | --- | --- | --- | -| `fast` | [StoreSpec](#storespec) | Yes | — | Fast store that will be attempted to be contacted before reaching out to the `slow` store. | +| `fast` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | Fast store that will be attempted to be contacted before reaching out to the `slow` store. | | `fast_direction` | [StoreDirection](#storedirection) | — | `"both"` | How to handle the fast store. This can be useful to set to Get for worker nodes such that results are persisted to the slow store only. | -| `slow` | [StoreSpec](#storespec) | Yes | — | If the object does not exist in the `fast` store it will try to get it from this store. | +| `slow` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | If the object does not exist in the `fast` store it will try to get it from this store. | | `slow_direction` | [StoreDirection](#storedirection) | — | `"both"` | How to handle the slow store. This can be useful if creating a diode and you wish to have an upstream read only store. | ## ShardSpec @@ -781,8 +781,8 @@ See [`experimental_cloud_object_store`](#experimental_cloud_object_store-1) for | Field | Type | Required | Default | Description | | --- | --- | --- | --- | --- | | `size` | integer (uint64) | Yes | — | Size to partition the data on. | -| `lower_store` | [StoreSpec](#storespec) | Yes | — | Store to send data when object is < (less than) size. | -| `upper_store` | [StoreSpec](#storespec) | Yes | — | Store to send data when object is >= (less than eq) size. | +| `lower_store` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | Store to send data when object is < (less than) size. | +| `upper_store` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | Store to send data when object is >= (less than eq) size. | ## GrpcSpec @@ -955,587 +955,6 @@ Listener for HTTP/HTTPS/HTTP2 sockets. | `"sha256"` | Use the sha256 hash function. [https://en.wikipedia.org/wiki/SHA-2](https://en.wikipedia.org/wiki/SHA-2) | | `"blake3"` | Use the blake3 hash function. [https://en.wikipedia.org/wiki/BLAKE_(hash_function)](https://en.wikipedia.org/wiki/BLAKE_(hash_function)) | -## StoreSpec - -Plus exactly one of the following variants (the key selects the variant): - -### `cache_metrics` - -Cache metrics store wraps another store and emits low-cardinality -OpenTelemetry cache operation metrics for the wrapped store. - -This wrapper is opt-in. Stores that are not explicitly wrapped by -`cache_metrics` are constructed exactly as they are without this -wrapper and do not pay its hot-path timing or recording cost. - -**Example JSON5 config:** -```json5 -"cache_metrics": { - "cache_type": "cas", - "backend": { - "filesystem": { - "content_path": "~/.cache/nativelink/content_path-cas", - "temp_path": "~/.cache/nativelink/tmp_path-cas" - } - } -} -``` - -**Type:** [CacheMetricsSpec](#cachemetricsspec) - -### `memory` - -Memory store will store all data in a hash map in memory. - -**Example JSON5 config:** -```json5 -"memory": { - "eviction_policy": { - "max_bytes": "10mb", - } -} -``` - -**Type:** [MemorySpec](#memoryspec) - -### `experimental_cloud_object_store` - -A generic blob store that will store files on the cloud -provider. This configuration will never delete files, so you are -responsible for purging old files in other ways. -It supports the following backends: - -1. **Amazon S3:** - S3 store will use Amazon's S3 service as a backend to store - the files. This configuration can be used to share files - across multiple instances. Uses system certificates for TLS - verification via `rustls-platform-verifier`. - - **Example JSON5 config:** - ```json5 - "experimental_cloud_object_store": { - "provider": "aws", - "region": "eu-north-1", - "bucket": "crossplane-bucket-af79aeca9", - "key_prefix": "test-prefix-index/", - "retry": { - "max_retries": 6, - "delay": 0.3, - "jitter": 0.5 - }, - "multipart_max_concurrent_uploads": 10 - } - ``` - -2. **Google Cloud Storage:** - GCS store uses Google's GCS service as a backend to store - the files. This configuration can be used to share files - across multiple instances. - - **Example JSON5 config:** - ```json5 - "experimental_cloud_object_store": { - "provider": "gcs", - "bucket": "test-bucket", - "key_prefix": "test-prefix-index/", - "retry": { - "max_retries": 6, - "delay": 0.3, - "jitter": 0.5 - }, - "multipart_max_concurrent_uploads": 10 - } - ``` - -3. **Azure Blob Store:** - Azure Blob store will use Microsoft's Azure Blob service as a - backend to store the files. This configuration can be used to - share files across multiple instances. - - **Example JSON5 config:** - ```json5 - "experimental_cloud_object_store": { - "provider": "azure", - "account_name": "cloudshell1393657559", - "container": "simple-test-container", - "key_prefix": "folder/", - "retry": { - "max_retries": 6, - "delay": 0.3, - "jitter": 0.5 - }, - "multipart_max_concurrent_uploads": 10 - } - ``` - -4. **`NetApp` ONTAP S3** - `NetApp` ONTAP S3 store will use ONTAP's S3-compatible storage as a backend - to store files. This store is specifically configured for ONTAP's S3 requirements - including custom TLS configuration, credentials management, and proper vserver - configuration. - - This store uses AWS environment variables for credentials: - - `AWS_ACCESS_KEY_ID` - - `AWS_SECRET_ACCESS_KEY` - - `AWS_DEFAULT_REGION` - - **Example JSON5 config:** - ```json5 - "experimental_cloud_object_store": { - "provider": "ontap", - "endpoint": "https://ontap-s3-endpoint:443", - "vserver_name": "your-vserver", - "bucket": "your-bucket", - "root_certificates": "/path/to/certs.pem", // Optional - "key_prefix": "test-prefix/", // Optional - "retry": { - "max_retries": 6, - "delay": 0.3, - "jitter": 0.5 - }, - "multipart_max_concurrent_uploads": 10 - } - ``` - -**Type:** [ExperimentalCloudObjectSpec](#experimentalcloudobjectspec) - -### `ontap_s3_existence_cache` - -ONTAP S3 Existence Cache provides a caching layer on top of the ONTAP S3 store -to optimize repeated existence checks. It maintains an in-memory cache of object -digests and periodically syncs this cache to disk for persistence. - -The cache helps reduce latency for repeated calls to check object existence, -while still ensuring eventual consistency with the underlying ONTAP S3 store. - -Example JSON5 config: -```json5 -"ontap_s3_existence_cache": { - "index_path": "/path/to/cache/index.json", - "sync_interval_seconds": 300, - "backend": { - "endpoint": "https://ontap-s3-endpoint:443", - "vserver_name": "your-vserver", - "bucket": "your-bucket", - "key_prefix": "test-prefix/" - } -} -``` - -**Type:** [OntapS3ExistenceCacheSpec](#ontaps3existencecachespec) - -### `verify` - -Verify store is used to apply verifications to an underlying -store implementation. It is strongly encouraged to validate -as much data as you can before accepting data from a client, -failing to do so may cause the data in the store to be -populated with invalid data causing all kinds of problems. - -The suggested configuration is to have the CAS validate the -hash and size and the AC validate nothing. - -**Example JSON5 config:** -```json5 -"verify": { - "backend": { - "memory": { - "eviction_policy": { - "max_bytes": "500mb" - } - }, - }, - "verify_size": true, - "verify_hash": true -} -``` - -**Type:** [VerifySpec](#verifyspec) - -### `completeness_checking` - -Completeness checking store verifies if the -output files & folders exist in the CAS before forwarding -the request to the underlying store. -Note: This store should only be used on AC stores. - -**Example JSON5 config:** -```json5 -"completeness_checking": { - "backend": { - "filesystem": { - "content_path": "~/.cache/nativelink/content_path-ac", - "temp_path": "~/.cache/nativelink/tmp_path-ac", - "eviction_policy": { - "max_bytes": "500mb", - } - } - }, - "cas_store": { - "ref_store": { - "name": "CAS_MAIN_STORE" - } - } -} -``` - -**Type:** [CompletenessCheckingSpec](#completenesscheckingspec) - -### `compression` - -A compression store that will compress the data inbound and -outbound. There will be a non-trivial cost to compress and -decompress the data, but in many cases if the final store is -a store that requires network transport and/or storage space -is a concern it is often faster and more efficient to use this -store before those stores. - -**Example JSON5 config:** -```json5 -"compression": { - "compression_algorithm": { - "lz4": {} - }, - "backend": { - "filesystem": { - "content_path": "/tmp/nativelink/data/content_path-cas", - "temp_path": "/tmp/nativelink/data/tmp_path-cas", - "eviction_policy": { - "max_bytes": "2gb", - } - } - } -} -``` - -**Type:** [CompressionSpec](#compressionspec) - -### `dedup` - -A dedup store will take the inputs and run a rolling hash -algorithm on them to slice the input into smaller parts then -run a sha256 algorithm on the slice and if the object doesn't -already exist, upload the slice to the `content_store` using -a new digest of just the slice. Once all parts exist, an -Action-Cache-like digest will be built and uploaded to the -`index_store` which will contain a reference to each -chunk/digest of the uploaded file. Downloading a request will -first grab the index from the `index_store`, and forward the -download content of each chunk as if it were one file. - -This store is exceptionally good when the following conditions -are met: -* Content is mostly the same (inserts, updates, deletes are ok) -* Content is not compressed or encrypted -* Uploading or downloading from `content_store` is the bottleneck. - -Note: This store pairs well when used with `CompressionSpec` as -the `content_store`, but never put `DedupSpec` as the backend of -`CompressionSpec` as it will negate all the gains. - -Note: When running `.has()` on this store, it will only check -to see if the entry exists in the `index_store` and not check -if the individual chunks exist in the `content_store`. - -**Example JSON5 config:** -```json5 -"dedup": { - "index_store": { - "memory": { - "eviction_policy": { - "max_bytes": "1GB", - } - } - }, - "content_store": { - "compression": { - "compression_algorithm": { - "lz4": {} - }, - "backend": { - "fast_slow": { - "fast": { - "memory": { - "eviction_policy": { - "max_bytes": "500MB", - } - } - }, - "slow": { - "filesystem": { - "content_path": "/tmp/nativelink/data/content_path-content", - "temp_path": "/tmp/nativelink/data/tmp_path-content", - "eviction_policy": { - "max_bytes": "2gb" - } - } - } - } - } - } - } -} -``` - -**Type:** [DedupSpec](#dedupspec) - -### `existence_cache` - -Existence store will wrap around another store and cache calls -to has so that subsequent `has_with_results` calls will be -faster. This is useful for cases when you have a store that -is slow to respond to has calls. -Note: This store should only be used on CAS stores. - -**Example JSON5 config:** -```json5 -"existence_cache": { - "backend": { - "memory": { - "eviction_policy": { - "max_bytes": "500mb", - } - } - }, - // Note this is the existence store policy, not the backend policy - "eviction_policy": { - "max_seconds": 100, - } -} -``` - -**Type:** [ExistenceCacheSpec](#existencecachespec) - -### `fast_slow` - -`FastSlow` store will first try to fetch the data from the `fast` -store and then if it does not exist try the `slow` store. -When the object does exist in the `slow` store, it will copy -the data to the `fast` store while returning the data. -This store should be thought of as a store that "buffers" -the data to the `fast` store. -On uploads it will mirror data to both `fast` and `slow` stores. - -WARNING: If you need data to always exist in the `slow` store -for something like remote execution, be careful because this -store will never check to see if the objects exist in the -`slow` store if it exists in the `fast` store (i.e., it assumes -that if an object exists in the `fast` store it will exist in -the `slow` store). - -***Example JSON5 config:*** -```json5 -"fast_slow": { - "fast": { - "filesystem": { - "content_path": "/tmp/nativelink/data/content_path-index", - "temp_path": "/tmp/nativelink/data/tmp_path-index", - "eviction_policy": { - "max_bytes": "500mb", - } - } - }, - "slow": { - "filesystem": { - "content_path": "/tmp/nativelink/data/content_path-index", - "temp_path": "/tmp/nativelink/data/tmp_path-index", - "eviction_policy": { - "max_bytes": "500mb", - } - } - } -} -``` - -**Type:** [FastSlowSpec](#fastslowspec) - -### `shard` - -Shards the data to multiple stores. This is useful for cases -when you want to distribute the load across multiple stores. -The digest hash is used to determine which store to send the -data to. - -**Example JSON5 config:** -```json5 -"shard": { - "stores": [ - { - "store": { - "memory": { - "eviction_policy": { - "max_bytes": "10mb" - }, - }, - }, - "weight": 1 - }] -} -``` - -**Type:** [ShardSpec](#shardspec) - -### `filesystem` - -Stores the data on the filesystem. This store is designed for -local persistent storage. Restarts of this program should restore -the previous state, meaning anything uploaded will be persistent -as long as the filesystem integrity holds. - -**Example JSON5 config:** -```json5 -"filesystem": { - "content_path": "/tmp/nativelink/data-worker-test/content_path-cas", - "temp_path": "/tmp/nativelink/data-worker-test/tmp_path-cas", - "eviction_policy": { - "max_bytes": "10gb", - } -} -``` - -**Type:** [FilesystemSpec](#filesystemspec) - -### `ref_store` - -Store used to reference a store in the root store manager. -This is useful for cases when you want to share a store in different -nested stores. Example, you may want to share the same memory store -used for the action cache, but use a `FastSlowSpec` and have the fast -store also share the memory store for efficiency. - -**Example JSON5 config:** -```json5 -"ref_store": { - "name": "FS_CONTENT_STORE" -} -``` - -**Type:** [RefSpec](#refspec) - -### `size_partitioning` - -Uses the size field of the digest to separate which store to send the -data. This is useful for cases when you'd like to put small objects -in one store and large objects in another store. This should only be -used if the size field is the real size of the content, in other -words, don't use on AC (Action Cache) stores. Any store where you can -safely use `VerifySpec.verify_size = true`, this store should be safe -to use (i.e., CAS stores). - -**Example JSON5 config:** -```json5 -"size_partitioning": { - "size": "128mib", - "lower_store": { - "memory": { - "eviction_policy": { - "max_bytes": "${NATIVELINK_CAS_MEMORY_CONTENT_LIMIT:-100mb}" - } - } - }, - "upper_store": { - /// This store discards data larger than 128mib. - "noop": {} - } -} -``` - -**Type:** [SizePartitioningSpec](#sizepartitioningspec) - -### `grpc` - -This store will pass-through calls to another GRPC store. This store -is not designed to be used as a sub-store of another store, but it -does satisfy the interface and will likely work. - -One major GOTCHA is that some stores use a special function on this -store to get the size of the underlying object, which is only reliable -when this store is serving the a CAS store, not an AC store. If using -this store directly without being a child of any store there are no -side effects and is the most efficient way to use it. - -**Example JSON5 config:** -```json5 -"grpc": { - "instance_name": "main", - "endpoints": [ - {"address": "grpc://${CAS_ENDPOINT:-127.0.0.1}:50051"} - ], - "connections_per_endpoint": "5", - "rpc_timeout_s": "5m", - "store_type": "ac", - // Static headers attached to every outgoing request to the upstream - // remote cache. Useful for fixed service-account credentials. - "headers": { - "authorization": "Bearer my-static-token" - }, - // Header names to copy from the inbound client request and forward to - // the upstream remote cache. Use this to pass through dynamic - // credentials such as a JWT sent by the build client. - "forward_headers": ["authorization", "x-custom-token"] -} -``` - -**Type:** [GrpcSpec](#grpcspec) - -### `redis_store` - -Stores data in any stores compatible with Redis APIs. - -Pairs well with `SizePartitioning` and/or `FastSlow` stores. -Ideal for accepting small object sizes as most Redis store -services have a max file upload of between 256Mb-512Mb. - -**Example JSON5 config:** -```json5 -"redis_store": { - "addresses": [ - "redis://127.0.0.1:6379/", - ], - "max_client_permits": 1000, -} -``` - -**Type:** [RedisSpec](#redisspec) - -### `noop` - -Noop store is a store that sends streams into the void and all data -retrieval will return 404 (`NotFound`). This can be useful for cases -where you may need to partition your data and part of your data needs -to be discarded. - -**Example JSON5 config:** -```json5 -"noop": {} -``` - -**Type:** [NoopSpec](#noopspec) - -### `experimental_mongo` - -Experimental `MongoDB` store implementation. - -This store uses `MongoDB` as a backend for storing data. It supports -both CAS (Content Addressable Storage) and scheduler data with -optional change streams for real-time updates. - -**Example JSON5 config:** -```json5 -"experimental_mongo": { - "connection_string": "mongodb://localhost:27017", - "database": "nativelink", - "cas_collection": "cas", - "key_prefix": "cas:", - "read_chunk_size": 65536, - "max_concurrent_uploads": 10, - "enable_change_streams": false, - "max_requests": "100" -} -``` - -**Type:** [ExperimentalMongoSpec](#experimentalmongospec) - ## EvictionPolicy Eviction policy always works on LRU (Least Recently Used). Any time an entry @@ -1630,7 +1049,7 @@ Configuration for an individual shard of the store. | Field | Type | Required | Default | Description | | --- | --- | --- | --- | --- | -| `store` | [StoreSpec](#storespec) | Yes | — | Store to shard the data to. | +| `store` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | Store to shard the data to. | | `weight` | integer (uint32) | — | 1 | The weight of the store. This is used to determine how much data should be sent to the store. The actual percentage is the sum of all the store's weights divided by the individual store's weight. | ## GrpcEndpoint diff --git a/web/apps/docs/content/docs/reference/nativelink-config/v1.4.0.mdx b/web/apps/docs/content/docs/reference/nativelink-config/v1.4.0.mdx index 789716942..43feca53e 100644 --- a/web/apps/docs/content/docs/reference/nativelink-config/v1.4.0.mdx +++ b/web/apps/docs/content/docs/reference/nativelink-config/v1.4.0.mdx @@ -688,7 +688,7 @@ Plus exactly one of the following variants (the key selects the variant): | Field | Type | Required | Default | Description | | --- | --- | --- | --- | --- | | `cache_type` | string | Yes | — | Low-cardinality cache type label for metrics, for example `cas` or `ac`. | -| `backend` | [StoreSpec](#storespec) | Yes | — | Store to wrap with cache operation metrics. | +| `backend` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | Store to wrap with cache operation metrics. | ## MemorySpec @@ -712,7 +712,7 @@ See [`experimental_cloud_object_store`](#experimental_cloud_object_store-1) for | Field | Type | Required | Default | Description | | --- | --- | --- | --- | --- | -| `backend` | [StoreSpec](#storespec) | Yes | — | The underlying store wrap around. All content will first flow through self before forwarding to backend. In the event there is an error detected in self, the connection to the backend will be terminated, and early termination should always cause updates to fail on the backend. | +| `backend` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | The underlying store wrap around. All content will first flow through self before forwarding to backend. In the event there is an error detected in self, the connection to the backend will be terminated, and early termination should always cause updates to fail on the backend. | | `verify_size` | boolean | — | `false` | If set the store will verify the size of the data before accepting an upload of data. | | `verify_hash` | boolean | — | `false` | If the data should be hashed and verify that the key matches the computed hash. The hash function is automatically determined based request and if not set will use the global default. | @@ -720,22 +720,22 @@ See [`experimental_cloud_object_store`](#experimental_cloud_object_store-1) for | Field | Type | Required | Default | Description | | --- | --- | --- | --- | --- | -| `backend` | [StoreSpec](#storespec) | Yes | — | The underlying store that will have it's results validated before sending to client. | -| `cas_store` | [StoreSpec](#storespec) | Yes | — | When a request is made, the results are decoded and all output digests/files are verified to exist in this CAS store before returning success. | +| `backend` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | The underlying store that will have it's results validated before sending to client. | +| `cas_store` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | When a request is made, the results are decoded and all output digests/files are verified to exist in this CAS store before returning success. | ## CompressionSpec | Field | Type | Required | Default | Description | | --- | --- | --- | --- | --- | -| `backend` | [StoreSpec](#storespec) | Yes | — | The underlying store wrap around. All content will first flow through self before forwarding to backend. In the event there is an error detected in self, the connection to the backend will be terminated, and early termination should always cause updates to fail on the backend. | +| `backend` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | The underlying store wrap around. All content will first flow through self before forwarding to backend. In the event there is an error detected in self, the connection to the backend will be terminated, and early termination should always cause updates to fail on the backend. | | `compression_algorithm` | [CompressionAlgorithm](#compressionalgorithm) | Yes | — | The compression algorithm to use. | ## DedupSpec | Field | Type | Required | Default | Description | | --- | --- | --- | --- | --- | -| `index_store` | [StoreSpec](#storespec) | Yes | — | Store used to store the index of each dedup slice. This store should generally be fast and small. | -| `content_store` | [StoreSpec](#storespec) | Yes | — | The store where the individual chunks will be uploaded. This store should generally be the slower & larger store. | +| `index_store` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | Store used to store the index of each dedup slice. This store should generally be fast and small. | +| `content_store` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | The store where the individual chunks will be uploaded. This store should generally be the slower & larger store. | | `min_size` | integer (uint32) | — | 65536 (64k) | Minimum size that a chunk will be when slicing up the content. Note: This setting can be increased to improve performance because it will actually not check this number of bytes when deciding where to partition the data. | | `normal_size` | integer (uint32) | — | 262144 (256k) | A best-effort attempt will be made to keep the average size of the chunks to this number. It is not a guarantee, but a slight attempt will be made. | | `max_size` | integer (uint32) | — | 524288 (512k) | Maximum size a chunk is allowed to be. | @@ -745,16 +745,16 @@ See [`experimental_cloud_object_store`](#experimental_cloud_object_store-1) for | Field | Type | Required | Default | Description | | --- | --- | --- | --- | --- | -| `backend` | [StoreSpec](#storespec) | Yes | — | The underlying store wrap around. All content will first flow through self before forwarding to backend. In the event there is an error detected in self, the connection to the backend will be terminated, and early termination should always cause updates to fail on the backend. | +| `backend` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | The underlying store wrap around. All content will first flow through self before forwarding to backend. In the event there is an error detected in self, the connection to the backend will be terminated, and early termination should always cause updates to fail on the backend. | | `eviction_policy` | [EvictionPolicy](#evictionpolicy) | — | — | Policy used to evict items out of the store. Failure to set this value will cause items to never be removed from the store causing infinite memory usage. | ## FastSlowSpec | Field | Type | Required | Default | Description | | --- | --- | --- | --- | --- | -| `fast` | [StoreSpec](#storespec) | Yes | — | Fast store that will be attempted to be contacted before reaching out to the `slow` store. | +| `fast` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | Fast store that will be attempted to be contacted before reaching out to the `slow` store. | | `fast_direction` | [StoreDirection](#storedirection) | — | `"both"` | How to handle the fast store. This can be useful to set to Get for worker nodes such that results are persisted to the slow store only. | -| `slow` | [StoreSpec](#storespec) | Yes | — | If the object does not exist in the `fast` store it will try to get it from this store. | +| `slow` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | If the object does not exist in the `fast` store it will try to get it from this store. | | `slow_direction` | [StoreDirection](#storedirection) | — | `"both"` | How to handle the slow store. This can be useful if creating a diode and you wish to have an upstream read only store. | ## ShardSpec @@ -785,8 +785,8 @@ See [`experimental_cloud_object_store`](#experimental_cloud_object_store-1) for | Field | Type | Required | Default | Description | | --- | --- | --- | --- | --- | | `size` | integer (uint64) | Yes | — | Size to partition the data on. | -| `lower_store` | [StoreSpec](#storespec) | Yes | — | Store to send data when object is < (less than) size. | -| `upper_store` | [StoreSpec](#storespec) | Yes | — | Store to send data when object is >= (less than eq) size. | +| `lower_store` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | Store to send data when object is < (less than) size. | +| `upper_store` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | Store to send data when object is >= (less than eq) size. | ## GrpcSpec @@ -969,587 +969,6 @@ Listener for HTTP/HTTPS/HTTP2 sockets. | `"sha256"` | Use the sha256 hash function. [https://en.wikipedia.org/wiki/SHA-2](https://en.wikipedia.org/wiki/SHA-2) | | `"blake3"` | Use the blake3 hash function. [https://en.wikipedia.org/wiki/BLAKE_(hash_function)](https://en.wikipedia.org/wiki/BLAKE_(hash_function)) | -## StoreSpec - -Plus exactly one of the following variants (the key selects the variant): - -### `cache_metrics` - -Cache metrics store wraps another store and emits low-cardinality -OpenTelemetry cache operation metrics for the wrapped store. - -This wrapper is opt-in. Stores that are not explicitly wrapped by -`cache_metrics` are constructed exactly as they are without this -wrapper and do not pay its hot-path timing or recording cost. - -**Example JSON5 config:** -```json5 -"cache_metrics": { - "cache_type": "cas", - "backend": { - "filesystem": { - "content_path": "~/.cache/nativelink/content_path-cas", - "temp_path": "~/.cache/nativelink/tmp_path-cas" - } - } -} -``` - -**Type:** [CacheMetricsSpec](#cachemetricsspec) - -### `memory` - -Memory store will store all data in a hash map in memory. - -**Example JSON5 config:** -```json5 -"memory": { - "eviction_policy": { - "max_bytes": "10mb", - } -} -``` - -**Type:** [MemorySpec](#memoryspec) - -### `experimental_cloud_object_store` - -A generic blob store that will store files on the cloud -provider. This configuration will never delete files, so you are -responsible for purging old files in other ways. -It supports the following backends: - -1. **Amazon S3:** - S3 store will use Amazon's S3 service as a backend to store - the files. This configuration can be used to share files - across multiple instances. Uses system certificates for TLS - verification via `rustls-platform-verifier`. - - **Example JSON5 config:** - ```json5 - "experimental_cloud_object_store": { - "provider": "aws", - "region": "eu-north-1", - "bucket": "crossplane-bucket-af79aeca9", - "key_prefix": "test-prefix-index/", - "retry": { - "max_retries": 6, - "delay": 0.3, - "jitter": 0.5 - }, - "multipart_max_concurrent_uploads": 10 - } - ``` - -2. **Google Cloud Storage:** - GCS store uses Google's GCS service as a backend to store - the files. This configuration can be used to share files - across multiple instances. - - **Example JSON5 config:** - ```json5 - "experimental_cloud_object_store": { - "provider": "gcs", - "bucket": "test-bucket", - "key_prefix": "test-prefix-index/", - "retry": { - "max_retries": 6, - "delay": 0.3, - "jitter": 0.5 - }, - "multipart_max_concurrent_uploads": 10 - } - ``` - -3. **Azure Blob Store:** - Azure Blob store will use Microsoft's Azure Blob service as a - backend to store the files. This configuration can be used to - share files across multiple instances. - - **Example JSON5 config:** - ```json5 - "experimental_cloud_object_store": { - "provider": "azure", - "account_name": "cloudshell1393657559", - "container": "simple-test-container", - "key_prefix": "folder/", - "retry": { - "max_retries": 6, - "delay": 0.3, - "jitter": 0.5 - }, - "multipart_max_concurrent_uploads": 10 - } - ``` - -4. **`NetApp` ONTAP S3** - `NetApp` ONTAP S3 store will use ONTAP's S3-compatible storage as a backend - to store files. This store is specifically configured for ONTAP's S3 requirements - including custom TLS configuration, credentials management, and proper vserver - configuration. - - This store uses AWS environment variables for credentials: - - `AWS_ACCESS_KEY_ID` - - `AWS_SECRET_ACCESS_KEY` - - `AWS_DEFAULT_REGION` - - **Example JSON5 config:** - ```json5 - "experimental_cloud_object_store": { - "provider": "ontap", - "endpoint": "https://ontap-s3-endpoint:443", - "vserver_name": "your-vserver", - "bucket": "your-bucket", - "root_certificates": "/path/to/certs.pem", // Optional - "key_prefix": "test-prefix/", // Optional - "retry": { - "max_retries": 6, - "delay": 0.3, - "jitter": 0.5 - }, - "multipart_max_concurrent_uploads": 10 - } - ``` - -**Type:** [ExperimentalCloudObjectSpec](#experimentalcloudobjectspec) - -### `ontap_s3_existence_cache` - -ONTAP S3 Existence Cache provides a caching layer on top of the ONTAP S3 store -to optimize repeated existence checks. It maintains an in-memory cache of object -digests and periodically syncs this cache to disk for persistence. - -The cache helps reduce latency for repeated calls to check object existence, -while still ensuring eventual consistency with the underlying ONTAP S3 store. - -Example JSON5 config: -```json5 -"ontap_s3_existence_cache": { - "index_path": "/path/to/cache/index.json", - "sync_interval_seconds": 300, - "backend": { - "endpoint": "https://ontap-s3-endpoint:443", - "vserver_name": "your-vserver", - "bucket": "your-bucket", - "key_prefix": "test-prefix/" - } -} -``` - -**Type:** [OntapS3ExistenceCacheSpec](#ontaps3existencecachespec) - -### `verify` - -Verify store is used to apply verifications to an underlying -store implementation. It is strongly encouraged to validate -as much data as you can before accepting data from a client, -failing to do so may cause the data in the store to be -populated with invalid data causing all kinds of problems. - -The suggested configuration is to have the CAS validate the -hash and size and the AC validate nothing. - -**Example JSON5 config:** -```json5 -"verify": { - "backend": { - "memory": { - "eviction_policy": { - "max_bytes": "500mb" - } - }, - }, - "verify_size": true, - "verify_hash": true -} -``` - -**Type:** [VerifySpec](#verifyspec) - -### `completeness_checking` - -Completeness checking store verifies if the -output files & folders exist in the CAS before forwarding -the request to the underlying store. -Note: This store should only be used on AC stores. - -**Example JSON5 config:** -```json5 -"completeness_checking": { - "backend": { - "filesystem": { - "content_path": "~/.cache/nativelink/content_path-ac", - "temp_path": "~/.cache/nativelink/tmp_path-ac", - "eviction_policy": { - "max_bytes": "500mb", - } - } - }, - "cas_store": { - "ref_store": { - "name": "CAS_MAIN_STORE" - } - } -} -``` - -**Type:** [CompletenessCheckingSpec](#completenesscheckingspec) - -### `compression` - -A compression store that will compress the data inbound and -outbound. There will be a non-trivial cost to compress and -decompress the data, but in many cases if the final store is -a store that requires network transport and/or storage space -is a concern it is often faster and more efficient to use this -store before those stores. - -**Example JSON5 config:** -```json5 -"compression": { - "compression_algorithm": { - "lz4": {} - }, - "backend": { - "filesystem": { - "content_path": "/tmp/nativelink/data/content_path-cas", - "temp_path": "/tmp/nativelink/data/tmp_path-cas", - "eviction_policy": { - "max_bytes": "2gb", - } - } - } -} -``` - -**Type:** [CompressionSpec](#compressionspec) - -### `dedup` - -A dedup store will take the inputs and run a rolling hash -algorithm on them to slice the input into smaller parts then -run a sha256 algorithm on the slice and if the object doesn't -already exist, upload the slice to the `content_store` using -a new digest of just the slice. Once all parts exist, an -Action-Cache-like digest will be built and uploaded to the -`index_store` which will contain a reference to each -chunk/digest of the uploaded file. Downloading a request will -first grab the index from the `index_store`, and forward the -download content of each chunk as if it were one file. - -This store is exceptionally good when the following conditions -are met: -* Content is mostly the same (inserts, updates, deletes are ok) -* Content is not compressed or encrypted -* Uploading or downloading from `content_store` is the bottleneck. - -Note: This store pairs well when used with `CompressionSpec` as -the `content_store`, but never put `DedupSpec` as the backend of -`CompressionSpec` as it will negate all the gains. - -Note: When running `.has()` on this store, it will only check -to see if the entry exists in the `index_store` and not check -if the individual chunks exist in the `content_store`. - -**Example JSON5 config:** -```json5 -"dedup": { - "index_store": { - "memory": { - "eviction_policy": { - "max_bytes": "1GB", - } - } - }, - "content_store": { - "compression": { - "compression_algorithm": { - "lz4": {} - }, - "backend": { - "fast_slow": { - "fast": { - "memory": { - "eviction_policy": { - "max_bytes": "500MB", - } - } - }, - "slow": { - "filesystem": { - "content_path": "/tmp/nativelink/data/content_path-content", - "temp_path": "/tmp/nativelink/data/tmp_path-content", - "eviction_policy": { - "max_bytes": "2gb" - } - } - } - } - } - } - } -} -``` - -**Type:** [DedupSpec](#dedupspec) - -### `existence_cache` - -Existence store will wrap around another store and cache calls -to has so that subsequent `has_with_results` calls will be -faster. This is useful for cases when you have a store that -is slow to respond to has calls. -Note: This store should only be used on CAS stores. - -**Example JSON5 config:** -```json5 -"existence_cache": { - "backend": { - "memory": { - "eviction_policy": { - "max_bytes": "500mb", - } - } - }, - // Note this is the existence store policy, not the backend policy - "eviction_policy": { - "max_seconds": 100, - } -} -``` - -**Type:** [ExistenceCacheSpec](#existencecachespec) - -### `fast_slow` - -`FastSlow` store will first try to fetch the data from the `fast` -store and then if it does not exist try the `slow` store. -When the object does exist in the `slow` store, it will copy -the data to the `fast` store while returning the data. -This store should be thought of as a store that "buffers" -the data to the `fast` store. -On uploads it will mirror data to both `fast` and `slow` stores. - -WARNING: If you need data to always exist in the `slow` store -for something like remote execution, be careful because this -store will never check to see if the objects exist in the -`slow` store if it exists in the `fast` store (i.e., it assumes -that if an object exists in the `fast` store it will exist in -the `slow` store). - -***Example JSON5 config:*** -```json5 -"fast_slow": { - "fast": { - "filesystem": { - "content_path": "/tmp/nativelink/data/content_path-index", - "temp_path": "/tmp/nativelink/data/tmp_path-index", - "eviction_policy": { - "max_bytes": "500mb", - } - } - }, - "slow": { - "filesystem": { - "content_path": "/tmp/nativelink/data/content_path-index", - "temp_path": "/tmp/nativelink/data/tmp_path-index", - "eviction_policy": { - "max_bytes": "500mb", - } - } - } -} -``` - -**Type:** [FastSlowSpec](#fastslowspec) - -### `shard` - -Shards the data to multiple stores. This is useful for cases -when you want to distribute the load across multiple stores. -The digest hash is used to determine which store to send the -data to. - -**Example JSON5 config:** -```json5 -"shard": { - "stores": [ - { - "store": { - "memory": { - "eviction_policy": { - "max_bytes": "10mb" - }, - }, - }, - "weight": 1 - }] -} -``` - -**Type:** [ShardSpec](#shardspec) - -### `filesystem` - -Stores the data on the filesystem. This store is designed for -local persistent storage. Restarts of this program should restore -the previous state, meaning anything uploaded will be persistent -as long as the filesystem integrity holds. - -**Example JSON5 config:** -```json5 -"filesystem": { - "content_path": "/tmp/nativelink/data-worker-test/content_path-cas", - "temp_path": "/tmp/nativelink/data-worker-test/tmp_path-cas", - "eviction_policy": { - "max_bytes": "10gb", - } -} -``` - -**Type:** [FilesystemSpec](#filesystemspec) - -### `ref_store` - -Store used to reference a store in the root store manager. -This is useful for cases when you want to share a store in different -nested stores. Example, you may want to share the same memory store -used for the action cache, but use a `FastSlowSpec` and have the fast -store also share the memory store for efficiency. - -**Example JSON5 config:** -```json5 -"ref_store": { - "name": "FS_CONTENT_STORE" -} -``` - -**Type:** [RefSpec](#refspec) - -### `size_partitioning` - -Uses the size field of the digest to separate which store to send the -data. This is useful for cases when you'd like to put small objects -in one store and large objects in another store. This should only be -used if the size field is the real size of the content, in other -words, don't use on AC (Action Cache) stores. Any store where you can -safely use `VerifySpec.verify_size = true`, this store should be safe -to use (i.e., CAS stores). - -**Example JSON5 config:** -```json5 -"size_partitioning": { - "size": "128mib", - "lower_store": { - "memory": { - "eviction_policy": { - "max_bytes": "${NATIVELINK_CAS_MEMORY_CONTENT_LIMIT:-100mb}" - } - } - }, - "upper_store": { - /// This store discards data larger than 128mib. - "noop": {} - } -} -``` - -**Type:** [SizePartitioningSpec](#sizepartitioningspec) - -### `grpc` - -This store will pass-through calls to another GRPC store. This store -is not designed to be used as a sub-store of another store, but it -does satisfy the interface and will likely work. - -One major GOTCHA is that some stores use a special function on this -store to get the size of the underlying object, which is only reliable -when this store is serving the a CAS store, not an AC store. If using -this store directly without being a child of any store there are no -side effects and is the most efficient way to use it. - -**Example JSON5 config:** -```json5 -"grpc": { - "instance_name": "main", - "endpoints": [ - {"address": "grpc://${CAS_ENDPOINT:-127.0.0.1}:50051"} - ], - "connections_per_endpoint": "5", - "rpc_timeout_s": "5m", - "store_type": "ac", - // Static headers attached to every outgoing request to the upstream - // remote cache. Useful for fixed service-account credentials. - "headers": { - "authorization": "Bearer my-static-token" - }, - // Header names to copy from the inbound client request and forward to - // the upstream remote cache. Use this to pass through dynamic - // credentials such as a JWT sent by the build client. - "forward_headers": ["authorization", "x-custom-token"] -} -``` - -**Type:** [GrpcSpec](#grpcspec) - -### `redis_store` - -Stores data in any stores compatible with Redis APIs. - -Pairs well with `SizePartitioning` and/or `FastSlow` stores. -Ideal for accepting small object sizes as most Redis store -services have a max file upload of between 256Mb-512Mb. - -**Example JSON5 config:** -```json5 -"redis_store": { - "addresses": [ - "redis://127.0.0.1:6379/", - ], - "max_client_permits": 1000, -} -``` - -**Type:** [RedisSpec](#redisspec) - -### `noop` - -Noop store is a store that sends streams into the void and all data -retrieval will return 404 (`NotFound`). This can be useful for cases -where you may need to partition your data and part of your data needs -to be discarded. - -**Example JSON5 config:** -```json5 -"noop": {} -``` - -**Type:** [NoopSpec](#noopspec) - -### `experimental_mongo` - -Experimental `MongoDB` store implementation. - -This store uses `MongoDB` as a backend for storing data. It supports -both CAS (Content Addressable Storage) and scheduler data with -optional change streams for real-time updates. - -**Example JSON5 config:** -```json5 -"experimental_mongo": { - "connection_string": "mongodb://localhost:27017", - "database": "nativelink", - "cas_collection": "cas", - "key_prefix": "cas:", - "read_chunk_size": 65536, - "max_concurrent_uploads": 10, - "enable_change_streams": false, - "max_requests": "100" -} -``` - -**Type:** [ExperimentalMongoSpec](#experimentalmongospec) - ## EvictionPolicy Eviction policy always works on LRU (Least Recently Used). Any time an entry @@ -1644,7 +1063,7 @@ Configuration for an individual shard of the store. | Field | Type | Required | Default | Description | | --- | --- | --- | --- | --- | -| `store` | [StoreSpec](#storespec) | Yes | — | Store to shard the data to. | +| `store` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | Store to shard the data to. | | `weight` | integer (uint32) | — | 1 | The weight of the store. This is used to determine how much data should be sent to the store. The actual percentage is the sum of all the store's weights divided by the individual store's weight. | ## GrpcEndpoint diff --git a/web/apps/docs/content/docs/reference/nativelink-config/v1.5.0.mdx b/web/apps/docs/content/docs/reference/nativelink-config/v1.5.0.mdx index f83656b30..0b11389b6 100644 --- a/web/apps/docs/content/docs/reference/nativelink-config/v1.5.0.mdx +++ b/web/apps/docs/content/docs/reference/nativelink-config/v1.5.0.mdx @@ -688,7 +688,7 @@ Plus exactly one of the following variants (the key selects the variant): | Field | Type | Required | Default | Description | | --- | --- | --- | --- | --- | | `cache_type` | string | Yes | — | Low-cardinality cache type label for metrics, for example `cas` or `ac`. | -| `backend` | [StoreSpec](#storespec) | Yes | — | Store to wrap with cache operation metrics. | +| `backend` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | Store to wrap with cache operation metrics. | ## MemorySpec @@ -712,7 +712,7 @@ See [`experimental_cloud_object_store`](#experimental_cloud_object_store-1) for | Field | Type | Required | Default | Description | | --- | --- | --- | --- | --- | -| `backend` | [StoreSpec](#storespec) | Yes | — | The underlying store wrap around. All content will first flow through self before forwarding to backend. In the event there is an error detected in self, the connection to the backend will be terminated, and early termination should always cause updates to fail on the backend. | +| `backend` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | The underlying store wrap around. All content will first flow through self before forwarding to backend. In the event there is an error detected in self, the connection to the backend will be terminated, and early termination should always cause updates to fail on the backend. | | `verify_size` | boolean | — | `false` | If set the store will verify the size of the data before accepting an upload of data. | | `verify_hash` | boolean | — | `false` | If the data should be hashed and verify that the key matches the computed hash. The hash function is automatically determined based request and if not set will use the global default. | @@ -720,22 +720,22 @@ See [`experimental_cloud_object_store`](#experimental_cloud_object_store-1) for | Field | Type | Required | Default | Description | | --- | --- | --- | --- | --- | -| `backend` | [StoreSpec](#storespec) | Yes | — | The underlying store that will have it's results validated before sending to client. | -| `cas_store` | [StoreSpec](#storespec) | Yes | — | When a request is made, the results are decoded and all output digests/files are verified to exist in this CAS store before returning success. | +| `backend` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | The underlying store that will have it's results validated before sending to client. | +| `cas_store` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | When a request is made, the results are decoded and all output digests/files are verified to exist in this CAS store before returning success. | ## CompressionSpec | Field | Type | Required | Default | Description | | --- | --- | --- | --- | --- | -| `backend` | [StoreSpec](#storespec) | Yes | — | The underlying store wrap around. All content will first flow through self before forwarding to backend. In the event there is an error detected in self, the connection to the backend will be terminated, and early termination should always cause updates to fail on the backend. | +| `backend` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | The underlying store wrap around. All content will first flow through self before forwarding to backend. In the event there is an error detected in self, the connection to the backend will be terminated, and early termination should always cause updates to fail on the backend. | | `compression_algorithm` | [CompressionAlgorithm](#compressionalgorithm) | Yes | — | The compression algorithm to use. | ## DedupSpec | Field | Type | Required | Default | Description | | --- | --- | --- | --- | --- | -| `index_store` | [StoreSpec](#storespec) | Yes | — | Store used to store the index of each dedup slice. This store should generally be fast and small. | -| `content_store` | [StoreSpec](#storespec) | Yes | — | The store where the individual chunks will be uploaded. This store should generally be the slower & larger store. | +| `index_store` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | Store used to store the index of each dedup slice. This store should generally be fast and small. | +| `content_store` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | The store where the individual chunks will be uploaded. This store should generally be the slower & larger store. | | `min_size` | integer (uint32) | — | 65536 (64k) | Minimum size that a chunk will be when slicing up the content. Note: This setting can be increased to improve performance because it will actually not check this number of bytes when deciding where to partition the data. | | `normal_size` | integer (uint32) | — | 262144 (256k) | A best-effort attempt will be made to keep the average size of the chunks to this number. It is not a guarantee, but a slight attempt will be made. | | `max_size` | integer (uint32) | — | 524288 (512k) | Maximum size a chunk is allowed to be. | @@ -745,16 +745,16 @@ See [`experimental_cloud_object_store`](#experimental_cloud_object_store-1) for | Field | Type | Required | Default | Description | | --- | --- | --- | --- | --- | -| `backend` | [StoreSpec](#storespec) | Yes | — | The underlying store wrap around. All content will first flow through self before forwarding to backend. In the event there is an error detected in self, the connection to the backend will be terminated, and early termination should always cause updates to fail on the backend. | +| `backend` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | The underlying store wrap around. All content will first flow through self before forwarding to backend. In the event there is an error detected in self, the connection to the backend will be terminated, and early termination should always cause updates to fail on the backend. | | `eviction_policy` | [EvictionPolicy](#evictionpolicy) | — | — | Policy used to evict items out of the store. Failure to set this value will cause items to never be removed from the store causing infinite memory usage. | ## FastSlowSpec | Field | Type | Required | Default | Description | | --- | --- | --- | --- | --- | -| `fast` | [StoreSpec](#storespec) | Yes | — | Fast store that will be attempted to be contacted before reaching out to the `slow` store. | +| `fast` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | Fast store that will be attempted to be contacted before reaching out to the `slow` store. | | `fast_direction` | [StoreDirection](#storedirection) | — | `"both"` | How to handle the fast store. This can be useful to set to Get for worker nodes such that results are persisted to the slow store only. | -| `slow` | [StoreSpec](#storespec) | Yes | — | If the object does not exist in the `fast` store it will try to get it from this store. | +| `slow` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | If the object does not exist in the `fast` store it will try to get it from this store. | | `slow_direction` | [StoreDirection](#storedirection) | — | `"both"` | How to handle the slow store. This can be useful if creating a diode and you wish to have an upstream read only store. | | `bypass_dedup_threshold_bytes` | integer (uint64) | — | `0` | Reads of blobs at or above this size skip the leader/follower dedup map and stream straight from the slow store without populating the fast tier. `0` (the default) disables the bypass: every read goes through dedup, matching the prior behaviour. Enable it by setting a threshold — 256 MiB is a reasonable starting point for backends where large-blob dedup is a net loss (followers tend to time out anyway), but the right value is workload-dependent. | @@ -786,8 +786,8 @@ See [`experimental_cloud_object_store`](#experimental_cloud_object_store-1) for | Field | Type | Required | Default | Description | | --- | --- | --- | --- | --- | | `size` | integer (uint64) | Yes | — | Size to partition the data on. | -| `lower_store` | [StoreSpec](#storespec) | Yes | — | Store to send data when object is < (less than) size. | -| `upper_store` | [StoreSpec](#storespec) | Yes | — | Store to send data when object is >= (less than eq) size. | +| `lower_store` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | Store to send data when object is < (less than) size. | +| `upper_store` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | Store to send data when object is >= (less than eq) size. | ## GrpcSpec @@ -970,587 +970,6 @@ Listener for HTTP/HTTPS/HTTP2 sockets. | `"sha256"` | Use the sha256 hash function. [https://en.wikipedia.org/wiki/SHA-2](https://en.wikipedia.org/wiki/SHA-2) | | `"blake3"` | Use the blake3 hash function. [https://en.wikipedia.org/wiki/BLAKE_(hash_function)](https://en.wikipedia.org/wiki/BLAKE_(hash_function)) | -## StoreSpec - -Plus exactly one of the following variants (the key selects the variant): - -### `cache_metrics` - -Cache metrics store wraps another store and emits low-cardinality -OpenTelemetry cache operation metrics for the wrapped store. - -This wrapper is opt-in. Stores that are not explicitly wrapped by -`cache_metrics` are constructed exactly as they are without this -wrapper and do not pay its hot-path timing or recording cost. - -**Example JSON5 config:** -```json5 -"cache_metrics": { - "cache_type": "cas", - "backend": { - "filesystem": { - "content_path": "~/.cache/nativelink/content_path-cas", - "temp_path": "~/.cache/nativelink/tmp_path-cas" - } - } -} -``` - -**Type:** [CacheMetricsSpec](#cachemetricsspec) - -### `memory` - -Memory store will store all data in a hash map in memory. - -**Example JSON5 config:** -```json5 -"memory": { - "eviction_policy": { - "max_bytes": "10mb", - } -} -``` - -**Type:** [MemorySpec](#memoryspec) - -### `experimental_cloud_object_store` - -A generic blob store that will store files on the cloud -provider. This configuration will never delete files, so you are -responsible for purging old files in other ways. -It supports the following backends: - -1. **Amazon S3:** - S3 store will use Amazon's S3 service as a backend to store - the files. This configuration can be used to share files - across multiple instances. Uses system certificates for TLS - verification via `rustls-platform-verifier`. - - **Example JSON5 config:** - ```json5 - "experimental_cloud_object_store": { - "provider": "aws", - "region": "eu-north-1", - "bucket": "crossplane-bucket-af79aeca9", - "key_prefix": "test-prefix-index/", - "retry": { - "max_retries": 6, - "delay": 0.3, - "jitter": 0.5 - }, - "multipart_max_concurrent_uploads": 10 - } - ``` - -2. **Google Cloud Storage:** - GCS store uses Google's GCS service as a backend to store - the files. This configuration can be used to share files - across multiple instances. - - **Example JSON5 config:** - ```json5 - "experimental_cloud_object_store": { - "provider": "gcs", - "bucket": "test-bucket", - "key_prefix": "test-prefix-index/", - "retry": { - "max_retries": 6, - "delay": 0.3, - "jitter": 0.5 - }, - "multipart_max_concurrent_uploads": 10 - } - ``` - -3. **Azure Blob Store:** - Azure Blob store will use Microsoft's Azure Blob service as a - backend to store the files. This configuration can be used to - share files across multiple instances. - - **Example JSON5 config:** - ```json5 - "experimental_cloud_object_store": { - "provider": "azure", - "account_name": "cloudshell1393657559", - "container": "simple-test-container", - "key_prefix": "folder/", - "retry": { - "max_retries": 6, - "delay": 0.3, - "jitter": 0.5 - }, - "multipart_max_concurrent_uploads": 10 - } - ``` - -4. **`NetApp` ONTAP S3** - `NetApp` ONTAP S3 store will use ONTAP's S3-compatible storage as a backend - to store files. This store is specifically configured for ONTAP's S3 requirements - including custom TLS configuration, credentials management, and proper vserver - configuration. - - This store uses AWS environment variables for credentials: - - `AWS_ACCESS_KEY_ID` - - `AWS_SECRET_ACCESS_KEY` - - `AWS_DEFAULT_REGION` - - **Example JSON5 config:** - ```json5 - "experimental_cloud_object_store": { - "provider": "ontap", - "endpoint": "https://ontap-s3-endpoint:443", - "vserver_name": "your-vserver", - "bucket": "your-bucket", - "root_certificates": "/path/to/certs.pem", // Optional - "key_prefix": "test-prefix/", // Optional - "retry": { - "max_retries": 6, - "delay": 0.3, - "jitter": 0.5 - }, - "multipart_max_concurrent_uploads": 10 - } - ``` - -**Type:** [ExperimentalCloudObjectSpec](#experimentalcloudobjectspec) - -### `ontap_s3_existence_cache` - -ONTAP S3 Existence Cache provides a caching layer on top of the ONTAP S3 store -to optimize repeated existence checks. It maintains an in-memory cache of object -digests and periodically syncs this cache to disk for persistence. - -The cache helps reduce latency for repeated calls to check object existence, -while still ensuring eventual consistency with the underlying ONTAP S3 store. - -Example JSON5 config: -```json5 -"ontap_s3_existence_cache": { - "index_path": "/path/to/cache/index.json", - "sync_interval_seconds": 300, - "backend": { - "endpoint": "https://ontap-s3-endpoint:443", - "vserver_name": "your-vserver", - "bucket": "your-bucket", - "key_prefix": "test-prefix/" - } -} -``` - -**Type:** [OntapS3ExistenceCacheSpec](#ontaps3existencecachespec) - -### `verify` - -Verify store is used to apply verifications to an underlying -store implementation. It is strongly encouraged to validate -as much data as you can before accepting data from a client, -failing to do so may cause the data in the store to be -populated with invalid data causing all kinds of problems. - -The suggested configuration is to have the CAS validate the -hash and size and the AC validate nothing. - -**Example JSON5 config:** -```json5 -"verify": { - "backend": { - "memory": { - "eviction_policy": { - "max_bytes": "500mb" - } - }, - }, - "verify_size": true, - "verify_hash": true -} -``` - -**Type:** [VerifySpec](#verifyspec) - -### `completeness_checking` - -Completeness checking store verifies if the -output files & folders exist in the CAS before forwarding -the request to the underlying store. -Note: This store should only be used on AC stores. - -**Example JSON5 config:** -```json5 -"completeness_checking": { - "backend": { - "filesystem": { - "content_path": "~/.cache/nativelink/content_path-ac", - "temp_path": "~/.cache/nativelink/tmp_path-ac", - "eviction_policy": { - "max_bytes": "500mb", - } - } - }, - "cas_store": { - "ref_store": { - "name": "CAS_MAIN_STORE" - } - } -} -``` - -**Type:** [CompletenessCheckingSpec](#completenesscheckingspec) - -### `compression` - -A compression store that will compress the data inbound and -outbound. There will be a non-trivial cost to compress and -decompress the data, but in many cases if the final store is -a store that requires network transport and/or storage space -is a concern it is often faster and more efficient to use this -store before those stores. - -**Example JSON5 config:** -```json5 -"compression": { - "compression_algorithm": { - "lz4": {} - }, - "backend": { - "filesystem": { - "content_path": "/tmp/nativelink/data/content_path-cas", - "temp_path": "/tmp/nativelink/data/tmp_path-cas", - "eviction_policy": { - "max_bytes": "2gb", - } - } - } -} -``` - -**Type:** [CompressionSpec](#compressionspec) - -### `dedup` - -A dedup store will take the inputs and run a rolling hash -algorithm on them to slice the input into smaller parts then -run a sha256 algorithm on the slice and if the object doesn't -already exist, upload the slice to the `content_store` using -a new digest of just the slice. Once all parts exist, an -Action-Cache-like digest will be built and uploaded to the -`index_store` which will contain a reference to each -chunk/digest of the uploaded file. Downloading a request will -first grab the index from the `index_store`, and forward the -download content of each chunk as if it were one file. - -This store is exceptionally good when the following conditions -are met: -* Content is mostly the same (inserts, updates, deletes are ok) -* Content is not compressed or encrypted -* Uploading or downloading from `content_store` is the bottleneck. - -Note: This store pairs well when used with `CompressionSpec` as -the `content_store`, but never put `DedupSpec` as the backend of -`CompressionSpec` as it will negate all the gains. - -Note: When running `.has()` on this store, it will only check -to see if the entry exists in the `index_store` and not check -if the individual chunks exist in the `content_store`. - -**Example JSON5 config:** -```json5 -"dedup": { - "index_store": { - "memory": { - "eviction_policy": { - "max_bytes": "1GB", - } - } - }, - "content_store": { - "compression": { - "compression_algorithm": { - "lz4": {} - }, - "backend": { - "fast_slow": { - "fast": { - "memory": { - "eviction_policy": { - "max_bytes": "500MB", - } - } - }, - "slow": { - "filesystem": { - "content_path": "/tmp/nativelink/data/content_path-content", - "temp_path": "/tmp/nativelink/data/tmp_path-content", - "eviction_policy": { - "max_bytes": "2gb" - } - } - } - } - } - } - } -} -``` - -**Type:** [DedupSpec](#dedupspec) - -### `existence_cache` - -Existence store will wrap around another store and cache calls -to has so that subsequent `has_with_results` calls will be -faster. This is useful for cases when you have a store that -is slow to respond to has calls. -Note: This store should only be used on CAS stores. - -**Example JSON5 config:** -```json5 -"existence_cache": { - "backend": { - "memory": { - "eviction_policy": { - "max_bytes": "500mb", - } - } - }, - // Note this is the existence store policy, not the backend policy - "eviction_policy": { - "max_seconds": 100, - } -} -``` - -**Type:** [ExistenceCacheSpec](#existencecachespec) - -### `fast_slow` - -`FastSlow` store will first try to fetch the data from the `fast` -store and then if it does not exist try the `slow` store. -When the object does exist in the `slow` store, it will copy -the data to the `fast` store while returning the data. -This store should be thought of as a store that "buffers" -the data to the `fast` store. -On uploads it will mirror data to both `fast` and `slow` stores. - -WARNING: If you need data to always exist in the `slow` store -for something like remote execution, be careful because this -store will never check to see if the objects exist in the -`slow` store if it exists in the `fast` store (i.e., it assumes -that if an object exists in the `fast` store it will exist in -the `slow` store). - -***Example JSON5 config:*** -```json5 -"fast_slow": { - "fast": { - "filesystem": { - "content_path": "/tmp/nativelink/data/content_path-index", - "temp_path": "/tmp/nativelink/data/tmp_path-index", - "eviction_policy": { - "max_bytes": "500mb", - } - } - }, - "slow": { - "filesystem": { - "content_path": "/tmp/nativelink/data/content_path-index", - "temp_path": "/tmp/nativelink/data/tmp_path-index", - "eviction_policy": { - "max_bytes": "500mb", - } - } - } -} -``` - -**Type:** [FastSlowSpec](#fastslowspec) - -### `shard` - -Shards the data to multiple stores. This is useful for cases -when you want to distribute the load across multiple stores. -The digest hash is used to determine which store to send the -data to. - -**Example JSON5 config:** -```json5 -"shard": { - "stores": [ - { - "store": { - "memory": { - "eviction_policy": { - "max_bytes": "10mb" - }, - }, - }, - "weight": 1 - }] -} -``` - -**Type:** [ShardSpec](#shardspec) - -### `filesystem` - -Stores the data on the filesystem. This store is designed for -local persistent storage. Restarts of this program should restore -the previous state, meaning anything uploaded will be persistent -as long as the filesystem integrity holds. - -**Example JSON5 config:** -```json5 -"filesystem": { - "content_path": "/tmp/nativelink/data-worker-test/content_path-cas", - "temp_path": "/tmp/nativelink/data-worker-test/tmp_path-cas", - "eviction_policy": { - "max_bytes": "10gb", - } -} -``` - -**Type:** [FilesystemSpec](#filesystemspec) - -### `ref_store` - -Store used to reference a store in the root store manager. -This is useful for cases when you want to share a store in different -nested stores. Example, you may want to share the same memory store -used for the action cache, but use a `FastSlowSpec` and have the fast -store also share the memory store for efficiency. - -**Example JSON5 config:** -```json5 -"ref_store": { - "name": "FS_CONTENT_STORE" -} -``` - -**Type:** [RefSpec](#refspec) - -### `size_partitioning` - -Uses the size field of the digest to separate which store to send the -data. This is useful for cases when you'd like to put small objects -in one store and large objects in another store. This should only be -used if the size field is the real size of the content, in other -words, don't use on AC (Action Cache) stores. Any store where you can -safely use `VerifySpec.verify_size = true`, this store should be safe -to use (i.e., CAS stores). - -**Example JSON5 config:** -```json5 -"size_partitioning": { - "size": "128mib", - "lower_store": { - "memory": { - "eviction_policy": { - "max_bytes": "${NATIVELINK_CAS_MEMORY_CONTENT_LIMIT:-100mb}" - } - } - }, - "upper_store": { - /// This store discards data larger than 128mib. - "noop": {} - } -} -``` - -**Type:** [SizePartitioningSpec](#sizepartitioningspec) - -### `grpc` - -This store will pass-through calls to another GRPC store. This store -is not designed to be used as a sub-store of another store, but it -does satisfy the interface and will likely work. - -One major GOTCHA is that some stores use a special function on this -store to get the size of the underlying object, which is only reliable -when this store is serving the a CAS store, not an AC store. If using -this store directly without being a child of any store there are no -side effects and is the most efficient way to use it. - -**Example JSON5 config:** -```json5 -"grpc": { - "instance_name": "main", - "endpoints": [ - {"address": "grpc://${CAS_ENDPOINT:-127.0.0.1}:50051"} - ], - "connections_per_endpoint": "5", - "rpc_timeout_s": "5m", - "store_type": "ac", - // Static headers attached to every outgoing request to the upstream - // remote cache. Useful for fixed service-account credentials. - "headers": { - "authorization": "Bearer my-static-token" - }, - // Header names to copy from the inbound client request and forward to - // the upstream remote cache. Use this to pass through dynamic - // credentials such as a JWT sent by the build client. - "forward_headers": ["authorization", "x-custom-token"] -} -``` - -**Type:** [GrpcSpec](#grpcspec) - -### `redis_store` - -Stores data in any stores compatible with Redis APIs. - -Pairs well with `SizePartitioning` and/or `FastSlow` stores. -Ideal for accepting small object sizes as most Redis store -services have a max file upload of between 256Mb-512Mb. - -**Example JSON5 config:** -```json5 -"redis_store": { - "addresses": [ - "redis://127.0.0.1:6379/", - ], - "max_client_permits": 1000, -} -``` - -**Type:** [RedisSpec](#redisspec) - -### `noop` - -Noop store is a store that sends streams into the void and all data -retrieval will return 404 (`NotFound`). This can be useful for cases -where you may need to partition your data and part of your data needs -to be discarded. - -**Example JSON5 config:** -```json5 -"noop": {} -``` - -**Type:** [NoopSpec](#noopspec) - -### `experimental_mongo` - -Experimental `MongoDB` store implementation. - -This store uses `MongoDB` as a backend for storing data. It supports -both CAS (Content Addressable Storage) and scheduler data with -optional change streams for real-time updates. - -**Example JSON5 config:** -```json5 -"experimental_mongo": { - "connection_string": "mongodb://localhost:27017", - "database": "nativelink", - "cas_collection": "cas", - "key_prefix": "cas:", - "read_chunk_size": 65536, - "max_concurrent_uploads": 10, - "enable_change_streams": false, - "max_requests": "100" -} -``` - -**Type:** [ExperimentalMongoSpec](#experimentalmongospec) - ## EvictionPolicy Eviction policy always works on LRU (Least Recently Used). Any time an entry @@ -1645,7 +1064,7 @@ Configuration for an individual shard of the store. | Field | Type | Required | Default | Description | | --- | --- | --- | --- | --- | -| `store` | [StoreSpec](#storespec) | Yes | — | Store to shard the data to. | +| `store` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | Store to shard the data to. | | `weight` | integer (uint32) | — | 1 | The weight of the store. This is used to determine how much data should be sent to the store. The actual percentage is the sum of all the store's weights divided by the individual store's weight. | ## GrpcEndpoint diff --git a/web/apps/docs/content/docs/reference/nativelink-config/v1.5.1.mdx b/web/apps/docs/content/docs/reference/nativelink-config/v1.5.1.mdx index 998dc3d6e..cc9447fdf 100644 --- a/web/apps/docs/content/docs/reference/nativelink-config/v1.5.1.mdx +++ b/web/apps/docs/content/docs/reference/nativelink-config/v1.5.1.mdx @@ -688,7 +688,7 @@ Plus exactly one of the following variants (the key selects the variant): | Field | Type | Required | Default | Description | | --- | --- | --- | --- | --- | | `cache_type` | string | Yes | — | Low-cardinality cache type label for metrics, for example `cas` or `ac`. | -| `backend` | [StoreSpec](#storespec) | Yes | — | Store to wrap with cache operation metrics. | +| `backend` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | Store to wrap with cache operation metrics. | ## MemorySpec @@ -712,7 +712,7 @@ See [`experimental_cloud_object_store`](#experimental_cloud_object_store-1) for | Field | Type | Required | Default | Description | | --- | --- | --- | --- | --- | -| `backend` | [StoreSpec](#storespec) | Yes | — | The underlying store wrap around. All content will first flow through self before forwarding to backend. In the event there is an error detected in self, the connection to the backend will be terminated, and early termination should always cause updates to fail on the backend. | +| `backend` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | The underlying store wrap around. All content will first flow through self before forwarding to backend. In the event there is an error detected in self, the connection to the backend will be terminated, and early termination should always cause updates to fail on the backend. | | `verify_size` | boolean | — | `false` | If set the store will verify the size of the data before accepting an upload of data. | | `verify_hash` | boolean | — | `false` | If the data should be hashed and verify that the key matches the computed hash. The hash function is automatically determined based request and if not set will use the global default. | @@ -720,22 +720,22 @@ See [`experimental_cloud_object_store`](#experimental_cloud_object_store-1) for | Field | Type | Required | Default | Description | | --- | --- | --- | --- | --- | -| `backend` | [StoreSpec](#storespec) | Yes | — | The underlying store that will have it's results validated before sending to client. | -| `cas_store` | [StoreSpec](#storespec) | Yes | — | When a request is made, the results are decoded and all output digests/files are verified to exist in this CAS store before returning success. | +| `backend` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | The underlying store that will have it's results validated before sending to client. | +| `cas_store` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | When a request is made, the results are decoded and all output digests/files are verified to exist in this CAS store before returning success. | ## CompressionSpec | Field | Type | Required | Default | Description | | --- | --- | --- | --- | --- | -| `backend` | [StoreSpec](#storespec) | Yes | — | The underlying store wrap around. All content will first flow through self before forwarding to backend. In the event there is an error detected in self, the connection to the backend will be terminated, and early termination should always cause updates to fail on the backend. | +| `backend` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | The underlying store wrap around. All content will first flow through self before forwarding to backend. In the event there is an error detected in self, the connection to the backend will be terminated, and early termination should always cause updates to fail on the backend. | | `compression_algorithm` | [CompressionAlgorithm](#compressionalgorithm) | Yes | — | The compression algorithm to use. | ## DedupSpec | Field | Type | Required | Default | Description | | --- | --- | --- | --- | --- | -| `index_store` | [StoreSpec](#storespec) | Yes | — | Store used to store the index of each dedup slice. This store should generally be fast and small. | -| `content_store` | [StoreSpec](#storespec) | Yes | — | The store where the individual chunks will be uploaded. This store should generally be the slower & larger store. | +| `index_store` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | Store used to store the index of each dedup slice. This store should generally be fast and small. | +| `content_store` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | The store where the individual chunks will be uploaded. This store should generally be the slower & larger store. | | `min_size` | integer (uint32) | — | 65536 (64k) | Minimum size that a chunk will be when slicing up the content. Note: This setting can be increased to improve performance because it will actually not check this number of bytes when deciding where to partition the data. | | `normal_size` | integer (uint32) | — | 262144 (256k) | A best-effort attempt will be made to keep the average size of the chunks to this number. It is not a guarantee, but a slight attempt will be made. | | `max_size` | integer (uint32) | — | 524288 (512k) | Maximum size a chunk is allowed to be. | @@ -745,16 +745,16 @@ See [`experimental_cloud_object_store`](#experimental_cloud_object_store-1) for | Field | Type | Required | Default | Description | | --- | --- | --- | --- | --- | -| `backend` | [StoreSpec](#storespec) | Yes | — | The underlying store wrap around. All content will first flow through self before forwarding to backend. In the event there is an error detected in self, the connection to the backend will be terminated, and early termination should always cause updates to fail on the backend. | +| `backend` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | The underlying store wrap around. All content will first flow through self before forwarding to backend. In the event there is an error detected in self, the connection to the backend will be terminated, and early termination should always cause updates to fail on the backend. | | `eviction_policy` | [EvictionPolicy](#evictionpolicy) | — | — | Policy used to evict items out of the store. Failure to set this value will cause items to never be removed from the store causing infinite memory usage. | ## FastSlowSpec | Field | Type | Required | Default | Description | | --- | --- | --- | --- | --- | -| `fast` | [StoreSpec](#storespec) | Yes | — | Fast store that will be attempted to be contacted before reaching out to the `slow` store. | +| `fast` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | Fast store that will be attempted to be contacted before reaching out to the `slow` store. | | `fast_direction` | [StoreDirection](#storedirection) | — | `"both"` | How to handle the fast store. This can be useful to set to Get for worker nodes such that results are persisted to the slow store only. | -| `slow` | [StoreSpec](#storespec) | Yes | — | If the object does not exist in the `fast` store it will try to get it from this store. | +| `slow` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | If the object does not exist in the `fast` store it will try to get it from this store. | | `slow_direction` | [StoreDirection](#storedirection) | — | `"both"` | How to handle the slow store. This can be useful if creating a diode and you wish to have an upstream read only store. | | `bypass_dedup_threshold_bytes` | integer (uint64) | — | `0` | Reads of blobs at or above this size skip the leader/follower dedup map and stream straight from the slow store without populating the fast tier. `0` (the default) disables the bypass: every read goes through dedup, matching the prior behaviour. Enable it by setting a threshold — 256 MiB is a reasonable starting point for backends where large-blob dedup is a net loss (followers tend to time out anyway), but the right value is workload-dependent. | @@ -786,8 +786,8 @@ See [`experimental_cloud_object_store`](#experimental_cloud_object_store-1) for | Field | Type | Required | Default | Description | | --- | --- | --- | --- | --- | | `size` | integer (uint64) | Yes | — | Size to partition the data on. | -| `lower_store` | [StoreSpec](#storespec) | Yes | — | Store to send data when object is < (less than) size. | -| `upper_store` | [StoreSpec](#storespec) | Yes | — | Store to send data when object is >= (less than eq) size. | +| `lower_store` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | Store to send data when object is < (less than) size. | +| `upper_store` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | Store to send data when object is >= (less than eq) size. | ## GrpcSpec @@ -970,587 +970,6 @@ Listener for HTTP/HTTPS/HTTP2 sockets. | `"sha256"` | Use the sha256 hash function. [https://en.wikipedia.org/wiki/SHA-2](https://en.wikipedia.org/wiki/SHA-2) | | `"blake3"` | Use the blake3 hash function. [https://en.wikipedia.org/wiki/BLAKE_(hash_function)](https://en.wikipedia.org/wiki/BLAKE_(hash_function)) | -## StoreSpec - -Plus exactly one of the following variants (the key selects the variant): - -### `cache_metrics` - -Cache metrics store wraps another store and emits low-cardinality -OpenTelemetry cache operation metrics for the wrapped store. - -This wrapper is opt-in. Stores that are not explicitly wrapped by -`cache_metrics` are constructed exactly as they are without this -wrapper and do not pay its hot-path timing or recording cost. - -**Example JSON5 config:** -```json5 -"cache_metrics": { - "cache_type": "cas", - "backend": { - "filesystem": { - "content_path": "~/.cache/nativelink/content_path-cas", - "temp_path": "~/.cache/nativelink/tmp_path-cas" - } - } -} -``` - -**Type:** [CacheMetricsSpec](#cachemetricsspec) - -### `memory` - -Memory store will store all data in a hash map in memory. - -**Example JSON5 config:** -```json5 -"memory": { - "eviction_policy": { - "max_bytes": "10mb", - } -} -``` - -**Type:** [MemorySpec](#memoryspec) - -### `experimental_cloud_object_store` - -A generic blob store that will store files on the cloud -provider. This configuration will never delete files, so you are -responsible for purging old files in other ways. -It supports the following backends: - -1. **Amazon S3:** - S3 store will use Amazon's S3 service as a backend to store - the files. This configuration can be used to share files - across multiple instances. Uses system certificates for TLS - verification via `rustls-platform-verifier`. - - **Example JSON5 config:** - ```json5 - "experimental_cloud_object_store": { - "provider": "aws", - "region": "eu-north-1", - "bucket": "crossplane-bucket-af79aeca9", - "key_prefix": "test-prefix-index/", - "retry": { - "max_retries": 6, - "delay": 0.3, - "jitter": 0.5 - }, - "multipart_max_concurrent_uploads": 10 - } - ``` - -2. **Google Cloud Storage:** - GCS store uses Google's GCS service as a backend to store - the files. This configuration can be used to share files - across multiple instances. - - **Example JSON5 config:** - ```json5 - "experimental_cloud_object_store": { - "provider": "gcs", - "bucket": "test-bucket", - "key_prefix": "test-prefix-index/", - "retry": { - "max_retries": 6, - "delay": 0.3, - "jitter": 0.5 - }, - "multipart_max_concurrent_uploads": 10 - } - ``` - -3. **Azure Blob Store:** - Azure Blob store will use Microsoft's Azure Blob service as a - backend to store the files. This configuration can be used to - share files across multiple instances. - - **Example JSON5 config:** - ```json5 - "experimental_cloud_object_store": { - "provider": "azure", - "account_name": "cloudshell1393657559", - "container": "simple-test-container", - "key_prefix": "folder/", - "retry": { - "max_retries": 6, - "delay": 0.3, - "jitter": 0.5 - }, - "multipart_max_concurrent_uploads": 10 - } - ``` - -4. **`NetApp` ONTAP S3** - `NetApp` ONTAP S3 store will use ONTAP's S3-compatible storage as a backend - to store files. This store is specifically configured for ONTAP's S3 requirements - including custom TLS configuration, credentials management, and proper vserver - configuration. - - This store uses AWS environment variables for credentials: - - `AWS_ACCESS_KEY_ID` - - `AWS_SECRET_ACCESS_KEY` - - `AWS_DEFAULT_REGION` - - **Example JSON5 config:** - ```json5 - "experimental_cloud_object_store": { - "provider": "ontap", - "endpoint": "https://ontap-s3-endpoint:443", - "vserver_name": "your-vserver", - "bucket": "your-bucket", - "root_certificates": "/path/to/certs.pem", // Optional - "key_prefix": "test-prefix/", // Optional - "retry": { - "max_retries": 6, - "delay": 0.3, - "jitter": 0.5 - }, - "multipart_max_concurrent_uploads": 10 - } - ``` - -**Type:** [ExperimentalCloudObjectSpec](#experimentalcloudobjectspec) - -### `ontap_s3_existence_cache` - -ONTAP S3 Existence Cache provides a caching layer on top of the ONTAP S3 store -to optimize repeated existence checks. It maintains an in-memory cache of object -digests and periodically syncs this cache to disk for persistence. - -The cache helps reduce latency for repeated calls to check object existence, -while still ensuring eventual consistency with the underlying ONTAP S3 store. - -Example JSON5 config: -```json5 -"ontap_s3_existence_cache": { - "index_path": "/path/to/cache/index.json", - "sync_interval_seconds": 300, - "backend": { - "endpoint": "https://ontap-s3-endpoint:443", - "vserver_name": "your-vserver", - "bucket": "your-bucket", - "key_prefix": "test-prefix/" - } -} -``` - -**Type:** [OntapS3ExistenceCacheSpec](#ontaps3existencecachespec) - -### `verify` - -Verify store is used to apply verifications to an underlying -store implementation. It is strongly encouraged to validate -as much data as you can before accepting data from a client, -failing to do so may cause the data in the store to be -populated with invalid data causing all kinds of problems. - -The suggested configuration is to have the CAS validate the -hash and size and the AC validate nothing. - -**Example JSON5 config:** -```json5 -"verify": { - "backend": { - "memory": { - "eviction_policy": { - "max_bytes": "500mb" - } - }, - }, - "verify_size": true, - "verify_hash": true -} -``` - -**Type:** [VerifySpec](#verifyspec) - -### `completeness_checking` - -Completeness checking store verifies if the -output files & folders exist in the CAS before forwarding -the request to the underlying store. -Note: This store should only be used on AC stores. - -**Example JSON5 config:** -```json5 -"completeness_checking": { - "backend": { - "filesystem": { - "content_path": "~/.cache/nativelink/content_path-ac", - "temp_path": "~/.cache/nativelink/tmp_path-ac", - "eviction_policy": { - "max_bytes": "500mb", - } - } - }, - "cas_store": { - "ref_store": { - "name": "CAS_MAIN_STORE" - } - } -} -``` - -**Type:** [CompletenessCheckingSpec](#completenesscheckingspec) - -### `compression` - -A compression store that will compress the data inbound and -outbound. There will be a non-trivial cost to compress and -decompress the data, but in many cases if the final store is -a store that requires network transport and/or storage space -is a concern it is often faster and more efficient to use this -store before those stores. - -**Example JSON5 config:** -```json5 -"compression": { - "compression_algorithm": { - "lz4": {} - }, - "backend": { - "filesystem": { - "content_path": "/tmp/nativelink/data/content_path-cas", - "temp_path": "/tmp/nativelink/data/tmp_path-cas", - "eviction_policy": { - "max_bytes": "2gb", - } - } - } -} -``` - -**Type:** [CompressionSpec](#compressionspec) - -### `dedup` - -A dedup store will take the inputs and run a rolling hash -algorithm on them to slice the input into smaller parts then -run a sha256 algorithm on the slice and if the object doesn't -already exist, upload the slice to the `content_store` using -a new digest of just the slice. Once all parts exist, an -Action-Cache-like digest will be built and uploaded to the -`index_store` which will contain a reference to each -chunk/digest of the uploaded file. Downloading a request will -first grab the index from the `index_store`, and forward the -download content of each chunk as if it were one file. - -This store is exceptionally good when the following conditions -are met: -* Content is mostly the same (inserts, updates, deletes are ok) -* Content is not compressed or encrypted -* Uploading or downloading from `content_store` is the bottleneck. - -Note: This store pairs well when used with `CompressionSpec` as -the `content_store`, but never put `DedupSpec` as the backend of -`CompressionSpec` as it will negate all the gains. - -Note: When running `.has()` on this store, it will only check -to see if the entry exists in the `index_store` and not check -if the individual chunks exist in the `content_store`. - -**Example JSON5 config:** -```json5 -"dedup": { - "index_store": { - "memory": { - "eviction_policy": { - "max_bytes": "1GB", - } - } - }, - "content_store": { - "compression": { - "compression_algorithm": { - "lz4": {} - }, - "backend": { - "fast_slow": { - "fast": { - "memory": { - "eviction_policy": { - "max_bytes": "500MB", - } - } - }, - "slow": { - "filesystem": { - "content_path": "/tmp/nativelink/data/content_path-content", - "temp_path": "/tmp/nativelink/data/tmp_path-content", - "eviction_policy": { - "max_bytes": "2gb" - } - } - } - } - } - } - } -} -``` - -**Type:** [DedupSpec](#dedupspec) - -### `existence_cache` - -Existence store will wrap around another store and cache calls -to has so that subsequent `has_with_results` calls will be -faster. This is useful for cases when you have a store that -is slow to respond to has calls. -Note: This store should only be used on CAS stores. - -**Example JSON5 config:** -```json5 -"existence_cache": { - "backend": { - "memory": { - "eviction_policy": { - "max_bytes": "500mb", - } - } - }, - // Note this is the existence store policy, not the backend policy - "eviction_policy": { - "max_seconds": 100, - } -} -``` - -**Type:** [ExistenceCacheSpec](#existencecachespec) - -### `fast_slow` - -`FastSlow` store will first try to fetch the data from the `fast` -store and then if it does not exist try the `slow` store. -When the object does exist in the `slow` store, it will copy -the data to the `fast` store while returning the data. -This store should be thought of as a store that "buffers" -the data to the `fast` store. -On uploads it will mirror data to both `fast` and `slow` stores. - -WARNING: If you need data to always exist in the `slow` store -for something like remote execution, be careful because this -store will never check to see if the objects exist in the -`slow` store if it exists in the `fast` store (i.e., it assumes -that if an object exists in the `fast` store it will exist in -the `slow` store). - -***Example JSON5 config:*** -```json5 -"fast_slow": { - "fast": { - "filesystem": { - "content_path": "/tmp/nativelink/data/content_path-index", - "temp_path": "/tmp/nativelink/data/tmp_path-index", - "eviction_policy": { - "max_bytes": "500mb", - } - } - }, - "slow": { - "filesystem": { - "content_path": "/tmp/nativelink/data/content_path-index", - "temp_path": "/tmp/nativelink/data/tmp_path-index", - "eviction_policy": { - "max_bytes": "500mb", - } - } - } -} -``` - -**Type:** [FastSlowSpec](#fastslowspec) - -### `shard` - -Shards the data to multiple stores. This is useful for cases -when you want to distribute the load across multiple stores. -The digest hash is used to determine which store to send the -data to. - -**Example JSON5 config:** -```json5 -"shard": { - "stores": [ - { - "store": { - "memory": { - "eviction_policy": { - "max_bytes": "10mb" - }, - }, - }, - "weight": 1 - }] -} -``` - -**Type:** [ShardSpec](#shardspec) - -### `filesystem` - -Stores the data on the filesystem. This store is designed for -local persistent storage. Restarts of this program should restore -the previous state, meaning anything uploaded will be persistent -as long as the filesystem integrity holds. - -**Example JSON5 config:** -```json5 -"filesystem": { - "content_path": "/tmp/nativelink/data-worker-test/content_path-cas", - "temp_path": "/tmp/nativelink/data-worker-test/tmp_path-cas", - "eviction_policy": { - "max_bytes": "10gb", - } -} -``` - -**Type:** [FilesystemSpec](#filesystemspec) - -### `ref_store` - -Store used to reference a store in the root store manager. -This is useful for cases when you want to share a store in different -nested stores. Example, you may want to share the same memory store -used for the action cache, but use a `FastSlowSpec` and have the fast -store also share the memory store for efficiency. - -**Example JSON5 config:** -```json5 -"ref_store": { - "name": "FS_CONTENT_STORE" -} -``` - -**Type:** [RefSpec](#refspec) - -### `size_partitioning` - -Uses the size field of the digest to separate which store to send the -data. This is useful for cases when you'd like to put small objects -in one store and large objects in another store. This should only be -used if the size field is the real size of the content, in other -words, don't use on AC (Action Cache) stores. Any store where you can -safely use `VerifySpec.verify_size = true`, this store should be safe -to use (i.e., CAS stores). - -**Example JSON5 config:** -```json5 -"size_partitioning": { - "size": "128mib", - "lower_store": { - "memory": { - "eviction_policy": { - "max_bytes": "${NATIVELINK_CAS_MEMORY_CONTENT_LIMIT:-100mb}" - } - } - }, - "upper_store": { - /// This store discards data larger than 128mib. - "noop": {} - } -} -``` - -**Type:** [SizePartitioningSpec](#sizepartitioningspec) - -### `grpc` - -This store will pass-through calls to another GRPC store. This store -is not designed to be used as a sub-store of another store, but it -does satisfy the interface and will likely work. - -One major GOTCHA is that some stores use a special function on this -store to get the size of the underlying object, which is only reliable -when this store is serving the a CAS store, not an AC store. If using -this store directly without being a child of any store there are no -side effects and is the most efficient way to use it. - -**Example JSON5 config:** -```json5 -"grpc": { - "instance_name": "main", - "endpoints": [ - {"address": "grpc://${CAS_ENDPOINT:-127.0.0.1}:50051"} - ], - "connections_per_endpoint": "5", - "rpc_timeout_s": "5m", - "store_type": "ac", - // Static headers attached to every outgoing request to the upstream - // remote cache. Useful for fixed service-account credentials. - "headers": { - "authorization": "Bearer my-static-token" - }, - // Header names to copy from the inbound client request and forward to - // the upstream remote cache. Use this to pass through dynamic - // credentials such as a JWT sent by the build client. - "forward_headers": ["authorization", "x-custom-token"] -} -``` - -**Type:** [GrpcSpec](#grpcspec) - -### `redis_store` - -Stores data in any stores compatible with Redis APIs. - -Pairs well with `SizePartitioning` and/or `FastSlow` stores. -Ideal for accepting small object sizes as most Redis store -services have a max file upload of between 256Mb-512Mb. - -**Example JSON5 config:** -```json5 -"redis_store": { - "addresses": [ - "redis://127.0.0.1:6379/", - ], - "max_client_permits": 1000, -} -``` - -**Type:** [RedisSpec](#redisspec) - -### `noop` - -Noop store is a store that sends streams into the void and all data -retrieval will return 404 (`NotFound`). This can be useful for cases -where you may need to partition your data and part of your data needs -to be discarded. - -**Example JSON5 config:** -```json5 -"noop": {} -``` - -**Type:** [NoopSpec](#noopspec) - -### `experimental_mongo` - -Experimental `MongoDB` store implementation. - -This store uses `MongoDB` as a backend for storing data. It supports -both CAS (Content Addressable Storage) and scheduler data with -optional change streams for real-time updates. - -**Example JSON5 config:** -```json5 -"experimental_mongo": { - "connection_string": "mongodb://localhost:27017", - "database": "nativelink", - "cas_collection": "cas", - "key_prefix": "cas:", - "read_chunk_size": 65536, - "max_concurrent_uploads": 10, - "enable_change_streams": false, - "max_requests": "100" -} -``` - -**Type:** [ExperimentalMongoSpec](#experimentalmongospec) - ## EvictionPolicy Eviction policy always works on LRU (Least Recently Used). Any time an entry @@ -1645,7 +1064,7 @@ Configuration for an individual shard of the store. | Field | Type | Required | Default | Description | | --- | --- | --- | --- | --- | -| `store` | [StoreSpec](#storespec) | Yes | — | Store to shard the data to. | +| `store` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | Store to shard the data to. | | `weight` | integer (uint32) | — | 1 | The weight of the store. This is used to determine how much data should be sent to the store. The actual percentage is the sum of all the store's weights divided by the individual store's weight. | ## GrpcEndpoint diff --git a/web/apps/docs/content/docs/reference/nativelink-config/v1.5.2.mdx b/web/apps/docs/content/docs/reference/nativelink-config/v1.5.2.mdx new file mode 100644 index 000000000..aa05f68ae --- /dev/null +++ b/web/apps/docs/content/docs/reference/nativelink-config/v1.5.2.mdx @@ -0,0 +1,1460 @@ +--- +title: Configuration reference +description: Every knob in the NativeLink JSON5 configuration — types, defaults, and links to source, autogenerated from the Rust config crate. +full: true +--- + +{/* AUTOGENERATED — do not edit by hand. + Source: nativelink-config @ v1.5.2 (6e63ef9a) + Regenerate from web/: bun --filter @nativelink/docs gen:config-reference */} + + + +This is the canonical NativeLink configuration reference for **v1.5.2**. +It is autogenerated from the Rust config crate +([`nativelink-config/src`](https://github.com/TraceMachina/nativelink/tree/v1.5.2/nativelink-config/src)) via the `build-schema` binary, so +it can never drift from what the binary actually deserializes. + +## Top-level fields + +The root object (`CasConfig`) accepts the following fields: + +| Field | Type | Required | Description | +| --- | --- | --- | --- | +| `stores` | array of [NamedStoreConfig](#namedstoreconfig) | Yes | List of stores available to use in this config. The keys can be used in other configs when needing to reference a store. | +| `workers` | array of [WorkerConfig](#workerconfig) | — | Worker configurations used to execute jobs. | +| `schedulers` | array of [NamedSchedulerConfig](#namedschedulerconfig) | — | List of schedulers available to use in this config. The keys can be used in other configs when needing to reference a scheduler. | +| `servers` | array of [ServerConfig](#serverconfig) | Yes | Servers to setup for this process. | +| `experimental_origin_events` | [OriginEventsSpec](#origineventsspec) | — | Experimental - Origin events configuration. This is the service that will collect and publish nativelink events to a store for processing by an external service. | +| `global` | [GlobalConfig](#globalconfig) | — | Any global configurations that apply to all modules live here. | + +## Configuration types + +Every type reachable from the root configuration, in reading order. + +## NamedStoreConfig + +**Common fields** + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `name` | string | Yes | — | | + +Plus exactly one of the following variants (the key selects the variant): + +### `cache_metrics` + +Cache metrics store wraps another store and emits low-cardinality +OpenTelemetry cache operation metrics for the wrapped store. + +This wrapper is opt-in. Stores that are not explicitly wrapped by +`cache_metrics` are constructed exactly as they are without this +wrapper and do not pay its hot-path timing or recording cost. + +**Example JSON5 config:** +```json5 +"cache_metrics": { + "cache_type": "cas", + "backend": { + "filesystem": { + "content_path": "~/.cache/nativelink/content_path-cas", + "temp_path": "~/.cache/nativelink/tmp_path-cas" + } + } +} +``` + +**Type:** [CacheMetricsSpec](#cachemetricsspec) + +### `memory` + +Memory store will store all data in a hash map in memory. + +**Example JSON5 config:** +```json5 +"memory": { + "eviction_policy": { + "max_bytes": "10mb", + } +} +``` + +**Type:** [MemorySpec](#memoryspec) + +### `experimental_cloud_object_store` + +A generic blob store that will store files on the cloud +provider. This configuration will never delete files, so you are +responsible for purging old files in other ways. +It supports the following backends: + +1. **Amazon S3:** + S3 store will use Amazon's S3 service as a backend to store + the files. This configuration can be used to share files + across multiple instances. Uses system certificates for TLS + verification via `rustls-platform-verifier`. + + **Example JSON5 config:** + ```json5 + "experimental_cloud_object_store": { + "provider": "aws", + "region": "eu-north-1", + "bucket": "crossplane-bucket-af79aeca9", + "key_prefix": "test-prefix-index/", + "retry": { + "max_retries": 6, + "delay": 0.3, + "jitter": 0.5 + }, + "multipart_max_concurrent_uploads": 10 + } + ``` + +2. **Google Cloud Storage:** + GCS store uses Google's GCS service as a backend to store + the files. This configuration can be used to share files + across multiple instances. + + **Example JSON5 config:** + ```json5 + "experimental_cloud_object_store": { + "provider": "gcs", + "bucket": "test-bucket", + "key_prefix": "test-prefix-index/", + "retry": { + "max_retries": 6, + "delay": 0.3, + "jitter": 0.5 + }, + "multipart_max_concurrent_uploads": 10 + } + ``` + +3. **Azure Blob Store:** + Azure Blob store will use Microsoft's Azure Blob service as a + backend to store the files. This configuration can be used to + share files across multiple instances. + + **Example JSON5 config:** + ```json5 + "experimental_cloud_object_store": { + "provider": "azure", + "account_name": "cloudshell1393657559", + "container": "simple-test-container", + "key_prefix": "folder/", + "retry": { + "max_retries": 6, + "delay": 0.3, + "jitter": 0.5 + }, + "multipart_max_concurrent_uploads": 10 + } + ``` + +4. **`NetApp` ONTAP S3** + `NetApp` ONTAP S3 store will use ONTAP's S3-compatible storage as a backend + to store files. This store is specifically configured for ONTAP's S3 requirements + including custom TLS configuration, credentials management, and proper vserver + configuration. + + This store uses AWS environment variables for credentials: + - `AWS_ACCESS_KEY_ID` + - `AWS_SECRET_ACCESS_KEY` + - `AWS_DEFAULT_REGION` + + **Example JSON5 config:** + ```json5 + "experimental_cloud_object_store": { + "provider": "ontap", + "endpoint": "https://ontap-s3-endpoint:443", + "vserver_name": "your-vserver", + "bucket": "your-bucket", + "root_certificates": "/path/to/certs.pem", // Optional + "key_prefix": "test-prefix/", // Optional + "retry": { + "max_retries": 6, + "delay": 0.3, + "jitter": 0.5 + }, + "multipart_max_concurrent_uploads": 10 + } + ``` + +**Type:** [ExperimentalCloudObjectSpec](#experimentalcloudobjectspec) + +### `ontap_s3_existence_cache` + +ONTAP S3 Existence Cache provides a caching layer on top of the ONTAP S3 store +to optimize repeated existence checks. It maintains an in-memory cache of object +digests and periodically syncs this cache to disk for persistence. + +The cache helps reduce latency for repeated calls to check object existence, +while still ensuring eventual consistency with the underlying ONTAP S3 store. + +Example JSON5 config: +```json5 +"ontap_s3_existence_cache": { + "index_path": "/path/to/cache/index.json", + "sync_interval_seconds": 300, + "backend": { + "endpoint": "https://ontap-s3-endpoint:443", + "vserver_name": "your-vserver", + "bucket": "your-bucket", + "key_prefix": "test-prefix/" + } +} +``` + +**Type:** [OntapS3ExistenceCacheSpec](#ontaps3existencecachespec) + +### `verify` + +Verify store is used to apply verifications to an underlying +store implementation. It is strongly encouraged to validate +as much data as you can before accepting data from a client, +failing to do so may cause the data in the store to be +populated with invalid data causing all kinds of problems. + +The suggested configuration is to have the CAS validate the +hash and size and the AC validate nothing. + +**Example JSON5 config:** +```json5 +"verify": { + "backend": { + "memory": { + "eviction_policy": { + "max_bytes": "500mb" + } + }, + }, + "verify_size": true, + "verify_hash": true +} +``` + +**Type:** [VerifySpec](#verifyspec) + +### `completeness_checking` + +Completeness checking store verifies if the +output files & folders exist in the CAS before forwarding +the request to the underlying store. +Note: This store should only be used on AC stores. + +**Example JSON5 config:** +```json5 +"completeness_checking": { + "backend": { + "filesystem": { + "content_path": "~/.cache/nativelink/content_path-ac", + "temp_path": "~/.cache/nativelink/tmp_path-ac", + "eviction_policy": { + "max_bytes": "500mb", + } + } + }, + "cas_store": { + "ref_store": { + "name": "CAS_MAIN_STORE" + } + } +} +``` + +**Type:** [CompletenessCheckingSpec](#completenesscheckingspec) + +### `compression` + +A compression store that will compress the data inbound and +outbound. There will be a non-trivial cost to compress and +decompress the data, but in many cases if the final store is +a store that requires network transport and/or storage space +is a concern it is often faster and more efficient to use this +store before those stores. + +**Example JSON5 config:** +```json5 +"compression": { + "compression_algorithm": { + "lz4": {} + }, + "backend": { + "filesystem": { + "content_path": "/tmp/nativelink/data/content_path-cas", + "temp_path": "/tmp/nativelink/data/tmp_path-cas", + "eviction_policy": { + "max_bytes": "2gb", + } + } + } +} +``` + +**Type:** [CompressionSpec](#compressionspec) + +### `dedup` + +A dedup store will take the inputs and run a rolling hash +algorithm on them to slice the input into smaller parts then +run a sha256 algorithm on the slice and if the object doesn't +already exist, upload the slice to the `content_store` using +a new digest of just the slice. Once all parts exist, an +Action-Cache-like digest will be built and uploaded to the +`index_store` which will contain a reference to each +chunk/digest of the uploaded file. Downloading a request will +first grab the index from the `index_store`, and forward the +download content of each chunk as if it were one file. + +This store is exceptionally good when the following conditions +are met: +* Content is mostly the same (inserts, updates, deletes are ok) +* Content is not compressed or encrypted +* Uploading or downloading from `content_store` is the bottleneck. + +Note: This store pairs well when used with `CompressionSpec` as +the `content_store`, but never put `DedupSpec` as the backend of +`CompressionSpec` as it will negate all the gains. + +Note: When running `.has()` on this store, it will only check +to see if the entry exists in the `index_store` and not check +if the individual chunks exist in the `content_store`. + +**Example JSON5 config:** +```json5 +"dedup": { + "index_store": { + "memory": { + "eviction_policy": { + "max_bytes": "1GB", + } + } + }, + "content_store": { + "compression": { + "compression_algorithm": { + "lz4": {} + }, + "backend": { + "fast_slow": { + "fast": { + "memory": { + "eviction_policy": { + "max_bytes": "500MB", + } + } + }, + "slow": { + "filesystem": { + "content_path": "/tmp/nativelink/data/content_path-content", + "temp_path": "/tmp/nativelink/data/tmp_path-content", + "eviction_policy": { + "max_bytes": "2gb" + } + } + } + } + } + } + } +} +``` + +**Type:** [DedupSpec](#dedupspec) + +### `existence_cache` + +Existence store will wrap around another store and cache calls +to has so that subsequent `has_with_results` calls will be +faster. This is useful for cases when you have a store that +is slow to respond to has calls. +Note: This store should only be used on CAS stores. + +**Example JSON5 config:** +```json5 +"existence_cache": { + "backend": { + "memory": { + "eviction_policy": { + "max_bytes": "500mb", + } + } + }, + // Note this is the existence store policy, not the backend policy + "eviction_policy": { + "max_seconds": 100, + } +} +``` + +**Type:** [ExistenceCacheSpec](#existencecachespec) + +### `fast_slow` + +`FastSlow` store will first try to fetch the data from the `fast` +store and then if it does not exist try the `slow` store. +When the object does exist in the `slow` store, it will copy +the data to the `fast` store while returning the data. +This store should be thought of as a store that "buffers" +the data to the `fast` store. +On uploads it will mirror data to both `fast` and `slow` stores. + +WARNING: If you need data to always exist in the `slow` store +for something like remote execution, be careful because this +store will never check to see if the objects exist in the +`slow` store if it exists in the `fast` store (i.e., it assumes +that if an object exists in the `fast` store it will exist in +the `slow` store). + +***Example JSON5 config:*** +```json5 +"fast_slow": { + "fast": { + "filesystem": { + "content_path": "/tmp/nativelink/data/content_path-index", + "temp_path": "/tmp/nativelink/data/tmp_path-index", + "eviction_policy": { + "max_bytes": "500mb", + } + } + }, + "slow": { + "filesystem": { + "content_path": "/tmp/nativelink/data/content_path-index", + "temp_path": "/tmp/nativelink/data/tmp_path-index", + "eviction_policy": { + "max_bytes": "500mb", + } + } + } +} +``` + +**Type:** [FastSlowSpec](#fastslowspec) + +### `shard` + +Shards the data to multiple stores. This is useful for cases +when you want to distribute the load across multiple stores. +The digest hash is used to determine which store to send the +data to. + +**Example JSON5 config:** +```json5 +"shard": { + "stores": [ + { + "store": { + "memory": { + "eviction_policy": { + "max_bytes": "10mb" + }, + }, + }, + "weight": 1 + }] +} +``` + +**Type:** [ShardSpec](#shardspec) + +### `filesystem` + +Stores the data on the filesystem. This store is designed for +local persistent storage. Restarts of this program should restore +the previous state, meaning anything uploaded will be persistent +as long as the filesystem integrity holds. + +**Example JSON5 config:** +```json5 +"filesystem": { + "content_path": "/tmp/nativelink/data-worker-test/content_path-cas", + "temp_path": "/tmp/nativelink/data-worker-test/tmp_path-cas", + "eviction_policy": { + "max_bytes": "10gb", + } +} +``` + +**Type:** [FilesystemSpec](#filesystemspec) + +### `ref_store` + +Store used to reference a store in the root store manager. +This is useful for cases when you want to share a store in different +nested stores. Example, you may want to share the same memory store +used for the action cache, but use a `FastSlowSpec` and have the fast +store also share the memory store for efficiency. + +**Example JSON5 config:** +```json5 +"ref_store": { + "name": "FS_CONTENT_STORE" +} +``` + +**Type:** [RefSpec](#refspec) + +### `size_partitioning` + +Uses the size field of the digest to separate which store to send the +data. This is useful for cases when you'd like to put small objects +in one store and large objects in another store. This should only be +used if the size field is the real size of the content, in other +words, don't use on AC (Action Cache) stores. Any store where you can +safely use `VerifySpec.verify_size = true`, this store should be safe +to use (i.e., CAS stores). + +**Example JSON5 config:** +```json5 +"size_partitioning": { + "size": "128mib", + "lower_store": { + "memory": { + "eviction_policy": { + "max_bytes": "${NATIVELINK_CAS_MEMORY_CONTENT_LIMIT:-100mb}" + } + } + }, + "upper_store": { + /// This store discards data larger than 128mib. + "noop": {} + } +} +``` + +**Type:** [SizePartitioningSpec](#sizepartitioningspec) + +### `grpc` + +This store will pass-through calls to another GRPC store. This store +is not designed to be used as a sub-store of another store, but it +does satisfy the interface and will likely work. + +One major GOTCHA is that some stores use a special function on this +store to get the size of the underlying object, which is only reliable +when this store is serving the a CAS store, not an AC store. If using +this store directly without being a child of any store there are no +side effects and is the most efficient way to use it. + +**Example JSON5 config:** +```json5 +"grpc": { + "instance_name": "main", + "endpoints": [ + {"address": "grpc://${CAS_ENDPOINT:-127.0.0.1}:50051"} + ], + "connections_per_endpoint": "5", + "rpc_timeout_s": "5m", + "store_type": "ac", + // Static headers attached to every outgoing request to the upstream + // remote cache. Useful for fixed service-account credentials. + "headers": { + "authorization": "Bearer my-static-token" + }, + // Header names to copy from the inbound client request and forward to + // the upstream remote cache. Use this to pass through dynamic + // credentials such as a JWT sent by the build client. + "forward_headers": ["authorization", "x-custom-token"] +} +``` + +**Type:** [GrpcSpec](#grpcspec) + +### `redis_store` + +Stores data in any stores compatible with Redis APIs. + +Pairs well with `SizePartitioning` and/or `FastSlow` stores. +Ideal for accepting small object sizes as most Redis store +services have a max file upload of between 256Mb-512Mb. + +**Example JSON5 config:** +```json5 +"redis_store": { + "addresses": [ + "redis://127.0.0.1:6379/", + ], + "max_client_permits": 1000, +} +``` + +**Type:** [RedisSpec](#redisspec) + +### `noop` + +Noop store is a store that sends streams into the void and all data +retrieval will return 404 (`NotFound`). This can be useful for cases +where you may need to partition your data and part of your data needs +to be discarded. + +**Example JSON5 config:** +```json5 +"noop": {} +``` + +**Type:** [NoopSpec](#noopspec) + +### `experimental_mongo` + +Experimental `MongoDB` store implementation. + +This store uses `MongoDB` as a backend for storing data. It supports +both CAS (Content Addressable Storage) and scheduler data with +optional change streams for real-time updates. + +**Example JSON5 config:** +```json5 +"experimental_mongo": { + "connection_string": "mongodb://localhost:27017", + "database": "nativelink", + "cas_collection": "cas", + "key_prefix": "cas:", + "read_chunk_size": 65536, + "max_concurrent_uploads": 10, + "enable_change_streams": false, + "max_requests": "100" +} +``` + +**Type:** [ExperimentalMongoSpec](#experimentalmongospec) + +## WorkerConfig + +Plus exactly one of the following variants (the key selects the variant): + +### `local` + +A worker type that executes jobs locally on this machine. + +**Type:** [LocalWorkerConfig](#localworkerconfig) + +## NamedSchedulerConfig + +**Common fields** + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `name` | string | Yes | — | | + +Plus exactly one of the following variants (the key selects the variant): + +### `simple` + +**Type:** [SimpleSpec](#simplespec) + +### `grpc` + +**Type:** [SchedulerGrpcSpec](#schedulergrpcspec) + +### `cache_lookup` + +**Type:** [CacheLookupSpec](#cachelookupspec) + +### `property_modifier` + +**Type:** [PropertyModifierSpec](#propertymodifierspec) + +### `historical_resource` + +**Type:** [HistoricalResourceSpec](#historicalresourcespec) + +## ServerConfig + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `name` | string | — | {index of server in config} | Name of the server. This is used to help identify the service for telemetry and logs. | +| `listener` | [ListenerConfig](#listenerconfig) | Yes | — | Configuration | +| `services` | [ServicesConfig](#servicesconfig) | — | — | Services to attach to server. | +| `experimental_identity_header` | [IdentityHeaderSpec](#identityheaderspec) | — | {see `IdentityHeaderSpec`} | The config related to identifying the client. | + +## OriginEventsSpec + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `publisher` | [OriginEventsPublisherSpec](#origineventspublisherspec) | Yes | — | The publisher configuration for origin events. | +| `max_event_queue_size` | integer (uint) | — | 65536 (zero defaults to this) | The maximum number of events to queue before applying back pressure. IMPORTANT: Backpressure causes all clients to slow down significantly. Zero is default. | + +## GlobalConfig + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `max_open_files` | integer (uint) | Yes | 24576 (= 24 * 1024) | Maximum number of open files that can be opened at one time. This value is not strictly enforced, it is a best effort. Some internal libraries open files or read metadata from a files which do not obey this limit, however the vast majority of cases will have this limit be honored. This value must be larger than `ulimit -n` to have any effect. Any network open file descriptors is not counted in this limit, but is counted in the kernel limit. It is a good idea to set a very large `ulimit -n`. Note: This value must be greater than 10. | +| `default_digest_hash_function` | [ConfigDigestHashFunction](#configdigesthashfunction) | — | `ConfigDigestHashFunction::sha256` | Default hash function to use while uploading blobs to the CAS when not set by client. | +| `default_digest_size_health_check` | integer (uint) | — | 1024*1024 (1MiB) | Default digest size to use for health check when running diagnostics checks. Health checks are expected to use this size for filling a buffer that is used for creation of digest. | + +## CacheMetricsSpec + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `cache_type` | string | Yes | — | Low-cardinality cache type label for metrics, for example `cas` or `ac`. | +| `backend` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | Store to wrap with cache operation metrics. | + +## MemorySpec + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `eviction_policy` | [EvictionPolicy](#evictionpolicy) | — | — | Policy used to evict items out of the store. Failure to set this value will cause items to never be removed from the store causing infinite memory usage. | + +## ExperimentalCloudObjectSpec + +See [`experimental_cloud_object_store`](#experimental_cloud_object_store-1) for details + +## OntapS3ExistenceCacheSpec + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `index_path` | string | Yes | — | | +| `sync_interval_seconds` | integer (uint32) | Yes | — | | +| `backend` | [ExperimentalOntapS3Spec](#experimentalontaps3spec) | Yes | — | | + +## VerifySpec + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `backend` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | The underlying store wrap around. All content will first flow through self before forwarding to backend. In the event there is an error detected in self, the connection to the backend will be terminated, and early termination should always cause updates to fail on the backend. | +| `verify_size` | boolean | — | `false` | If set the store will verify the size of the data before accepting an upload of data. | +| `verify_hash` | boolean | — | `false` | If the data should be hashed and verify that the key matches the computed hash. The hash function is automatically determined based request and if not set will use the global default. | + +## CompletenessCheckingSpec + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `backend` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | The underlying store that will have it's results validated before sending to client. | +| `cas_store` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | When a request is made, the results are decoded and all output digests/files are verified to exist in this CAS store before returning success. | + +## CompressionSpec + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `backend` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | The underlying store wrap around. All content will first flow through self before forwarding to backend. In the event there is an error detected in self, the connection to the backend will be terminated, and early termination should always cause updates to fail on the backend. | +| `compression_algorithm` | [CompressionAlgorithm](#compressionalgorithm) | Yes | — | The compression algorithm to use. | + +## DedupSpec + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `index_store` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | Store used to store the index of each dedup slice. This store should generally be fast and small. | +| `content_store` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | The store where the individual chunks will be uploaded. This store should generally be the slower & larger store. | +| `min_size` | integer (uint32) | — | 64k | Minimum size that a chunk will be when slicing up the content. Note: This setting can be increased to improve performance because it will actually not check this number of bytes when deciding where to partition the data. | +| `normal_size` | integer (uint32) | — | 256k | A best-effort attempt will be made to keep the average size of the chunks to this number. It is not a guarantee, but a slight attempt will be made. | +| `max_size` | integer (uint32) | — | 512k | Maximum size a chunk is allowed to be. | +| `max_concurrent_fetch_per_get` | integer (uint32) | — | 10 | Due to implementation detail, we want to prefer to download the first chunks of the file so we can stream the content out and free up some of our buffers. This configuration will be used to restrict the number of concurrent chunk downloads at a time per `get()` request. | + +## ExistenceCacheSpec + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `backend` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | The underlying store wrap around. All content will first flow through self before forwarding to backend. In the event there is an error detected in self, the connection to the backend will be terminated, and early termination should always cause updates to fail on the backend. | +| `eviction_policy` | [EvictionPolicy](#evictionpolicy) | — | — | Policy used to evict items out of the store. Failure to set this value will cause items to never be removed from the store causing infinite memory usage. | + +## FastSlowSpec + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `fast` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | Fast store that will be attempted to be contacted before reaching out to the `slow` store. | +| `fast_direction` | [StoreDirection](#storedirection) | — | `"both"` | How to handle the fast store. This can be useful to set to Get for worker nodes such that results are persisted to the slow store only. | +| `slow` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | If the object does not exist in the `fast` store it will try to get it from this store. | +| `slow_direction` | [StoreDirection](#storedirection) | — | `"both"` | How to handle the slow store. This can be useful if creating a diode and you wish to have an upstream read only store. | +| `bypass_dedup_threshold_bytes` | integer (uint64) | — | disabled (0) | Reads of blobs at or above this size skip the leader/follower dedup map and stream straight from the slow store without populating the fast tier. `0` (the default) disables the bypass: every read goes through dedup, matching the prior behaviour. Enable it by setting a threshold — 256 MiB is a reasonable starting point for backends where large-blob dedup is a net loss (followers tend to time out anyway), but the right value is workload-dependent. | + +## ShardSpec + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `stores` | array of [ShardConfig](#shardconfig) | Yes | — | Stores to shard the data to. | + +## FilesystemSpec + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `content_path` | string | Yes | — | Path on the system where to store the actual content. This is where the bulk of the data will be placed. On service startup this folder will be scanned and all files will be added to the cache. In the event one of the files doesn't match the criteria, the file will be deleted. | +| `temp_path` | string | Yes | — | A temporary location of where files that are being uploaded or deleted will be placed while the content cannot be guaranteed to be accurate. This location must be on the same block device as `content_path` so atomic moves can happen (i.e., move without copy). All files in this folder will be deleted on every startup. | +| `read_buffer_size` | integer (uint32) | — | 32k | Buffer size to use when reading files. Generally this should be left to the default value except for testing. | +| `eviction_policy` | [EvictionPolicy](#evictionpolicy) | — | — | Policy used to evict items out of the store. Failure to set this value will cause items to never be removed from the store causing infinite memory usage. | +| `block_size` | integer (uint64) | — | 4kb | The block size of the filesystem for the running machine value is used to determine an entry's actual size on disk consumed For a 4KB block size filesystem, a 1B file actually consumes 4KB | +| `max_concurrent_writes` | integer (uint) | — | unlimited | Maximum number of concurrent write operations allowed. Each write involves streaming data to a temp file and calling `sync_all()`, which can saturate disk I/O when many writes happen simultaneously. Limiting concurrency prevents disk saturation from blocking the async runtime. A value of 0 means unlimited (no concurrency limit). | + +## RefSpec + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `name` | string | Yes | — | Name of the store under the root "stores" config object. | + +## SizePartitioningSpec + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `size` | integer (uint64) | Yes | — | Size to partition the data on. | +| `lower_store` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | Store to send data when object is < (less than) size. | +| `upper_store` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | Store to send data when object is >= (less than eq) size. | + +## GrpcSpec + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `instance_name` | string | — | `""` | Instance name for GRPC calls. Proxy calls will have the `instance_name` changed to this. | +| `endpoints` | array of [GrpcEndpoint](#grpcendpoint) | Yes | — | The endpoint of the grpc connection. | +| `store_type` | [StoreType](#storetype) | Yes | — | The type of the upstream store, this ensures that the correct server calls are made. | +| `retry` | [Retry](#retry) | — | — | Retry configuration to use when a network request fails. | +| `max_concurrent_requests` | integer (uint) | — | `0` | Limit the number of simultaneous upstream requests to this many. A value of zero is treated as unlimited. If the limit is reached the request is queued. | +| `connections_per_endpoint` | integer (uint) | — | `0` | The number of connections to make to each specified endpoint to balance the load over multiple TCP connections. Default 1. | +| `rpc_timeout_s` | integer (uint64) | — | 0 (disabled) | Maximum time (seconds) allowed for a single RPC request (e.g. a `ByteStream.Write` call) before it is cancelled. | +| `use_legacy_resource_names` | boolean | — | false | Use legacy `ByteStream` resource name format, omitting the digest function component from the path. | +| `headers` | map of string to string | — | — | Static headers to attach to every outgoing gRPC request sent to this store's upstream endpoints. Useful for fixed authentication tokens (e.g. `{"authorization": "Bearer "}`) and other static metadata. | +| `forward_headers` | array of string | — | — | Header names to forward from the incoming client request to every outgoing upstream request. The header value is taken from the client request that triggered this store operation. Use this to pass through dynamic credentials such as JWT tokens sent by build clients. | + +## RedisSpec + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `addresses` | array of string | Yes | — | The hostname or IP address of the Redis server. Ex: `["redis://username:password@redis-server-url:6380/99"]` 99 Represents database ID, 6380 represents the port. | +| `response_timeout_s` | integer (uint64) | — | 10 | DEPRECATED: use `command_timeout_ms` The response timeout for the Redis connection in seconds. | +| `connection_timeout_s` | integer (uint64) | — | 10 | DEPRECATED: use `connection_timeout_ms` | +| `experimental_pub_sub_channel` | string | — | (Empty String / No Channel) | An optional and experimental Redis channel to publish write events to. | +| `key_prefix` | string | — | (Empty String / No Prefix) | An optional prefix to prepend to all keys in this store. | +| `mode` | [RedisMode](#redismode) | — | standard, | Set the mode Redis is operating in. | +| `broadcast_channel_capacity` | integer (uint) | — | `0` | Deprecated as redis-rs doesn't use it | +| `command_timeout_ms` | integer (uint64) | — | 10000 (10 seconds) | The amount of time in milliseconds until the Redis store considers the command to be timed out. This will trigger a retry of the command and potentially a reconnection to the Redis server. | +| `connection_timeout_ms` | integer (uint64) | — | 3000 (3 seconds) | The amount of time in milliseconds until the Redis store considers the connection to unresponsive. This will trigger a reconnection to the Redis server. | +| `health_check_timeout_ms` | integer (uint64) | — | 4000 (4 seconds) | Per-call ceiling for the `check_health` PING in milliseconds. | +| `read_chunk_size` | integer (uint) | — | 64KiB | The amount of data to read from the Redis server at a time. This is used to limit the amount of memory used when reading large objects from the Redis server as well as limiting the amount of time a single read operation can take. | +| `connection_pool_size` | integer (uint) | — | 3 | The number of connections to keep open to the Redis servers. | +| `max_chunk_uploads_per_update` | integer (uint) | — | 10 | The maximum number of upload chunks to allow per update. This is used to limit the amount of memory used when uploading large objects to the Redis server. A good rule of thumb is to think of the data as: `AVAIL_MEMORY / (read_chunk_size * max_chunk_uploads_per_update) = THORETICAL_MAX_CONCURRENT_UPLOADS` (note: it is a good idea to divide `AVAIL_MAX_MEMORY` by ~10 to account for other memory usage) | +| `scan_count` | integer (uint) | — | 10000 | The COUNT value passed when scanning keys in Redis. This is used to hint the amount of work that should be done per response. | +| `retry` | [Retry](#retry) | — | — | Retry configuration to use when a network request fails. | +| `max_client_permits` | integer (uint) | — | 500 | Maximum number of permitted actions to the Redis store at any one time This stops problems with timeouts due to many, many inflight actions | +| `max_count_per_cursor` | integer (uint64) | — | 1500 | Maximum number of items returned per cursor for the search indexes May reduce thundering herd issues with worker provisioner at higher node counts, | + +## NoopSpec + +_No fields._ + +## ExperimentalMongoSpec + +Configuration for `ExperimentalMongoDB` store. + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `connection_string` | string | Yes | — | `ExperimentalMongoDB` connection string. Example: <mongodb://localhost:27017> or <mongodb+srv://cluster.mongodb.net> | +| `database` | string | — | "nativelink" | The database name to use. | +| `cas_collection` | string | — | "cas" | The collection name for CAS data. | +| `scheduler_collection` | string | — | "scheduler" | The collection name for scheduler data. | +| `key_prefix` | string | — | "" | Prefix to prepend to all keys stored in `MongoDB`. | +| `read_chunk_size` | integer (uint) | — | 65536 (64KB) | The maximum amount of data to read from `MongoDB` in a single chunk (in bytes). | +| `max_concurrent_uploads` | integer (uint) | — | 10 | Deprecated, unused Maximum number of concurrent uploads allowed. | +| `connection_timeout_ms` | integer (uint64) | — | 3000 | Connection timeout in milliseconds. | +| `command_timeout_ms` | integer (uint64) | — | 10000 | Command timeout in milliseconds. | +| `enable_change_streams` | boolean | — | false | Enable `MongoDB` change streams for real-time updates. Required for scheduler subscriptions. | +| `write_concern_w` | string | — | — | Write concern 'w' parameter. Can be a number (e.g., 1) or string (e.g., "majority"). | +| `write_concern_j` | boolean | — | — | Write concern 'j' parameter (journal acknowledgment). | +| `write_concern_timeout_ms` | integer (uint32) | — | — | Write concern timeout in milliseconds. | +| `max_requests` | integer (uint) | — | Unlimited | Limits the number of requests at any one time | + +## LocalWorkerConfig + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `name` | string | — | {Index position in the workers list} | Name of the worker. This is give a more friendly name to a worker for logging and metric publishing. This is also the prefix of the worker id (i.e., "{name}{uuidv6}"). | +| `worker_api_endpoint` | [EndpointConfig](#endpointconfig) | Yes | — | Endpoint which the worker will connect to the scheduler's `WorkerApiService`. | +| `max_action_timeout` | integer (uint) | — | 20 minutes | The maximum time an action is allowed to run. If a task requests for a timeout longer than this time limit, the task will be rejected. Value in seconds. | +| `max_upload_timeout` | integer (uint) | — | 10 minutes | Maximum time allowed for uploading action results to CAS after execution completes. If upload takes longer than this, the action fails with `DeadlineExceeded` and may be retried by the scheduler. Value in seconds. | +| `max_inflight_tasks` | integer (uint64) | — | 0 (infinite tasks) | Maximum number of inflight tasks this worker can cope with. | +| `timeout_handled_externally` | boolean | — | false (`NativeLink` fully handles timeouts) | If timeout is handled in `entrypoint` or another wrapper script. If set to true `NativeLink` will not honor the timeout the action requested and instead will always force kill the action after `max_action_timeout` has been reached. If this is set to false, the smaller value of the action's timeout and `max_action_timeout` will be used to which `NativeLink` will kill the action. | +| `entrypoint` | string | — | {Use the command from the job request} | The command to execute on every execution request. This will be parsed as a command + arguments (not shell). Example: "run.sh" and a job with command: "sleep 5" will result in a command like: "run.sh sleep 5". | +| `experimental_precondition_script` | string | — | — | An optional script to run before every action is processed on the worker. The value should be the full path to the script to execute and will pause all actions on the worker if it returns an exit code other than 0. If not set, then the worker will never pause and will continue to accept jobs according to the scheduler configuration. This is useful, for example, if the worker should not take any more actions until there is enough resource available on the machine to handle them. | +| `cas_fast_slow_store` | string | Yes | — | Underlying CAS store that the worker will use to download CAS artifacts. This store must be a `FastSlowStore`. The `fast` store must be a `FileSystemStore` because it will use hardlinks when building out the files instead of copying the files. The slow store must eventually resolve to the same store the scheduler/client uses to send job requests. | +| `upload_action_result` | [UploadActionResultConfig](#uploadactionresultconfig) | — | — | Configuration for uploading action results. | +| `work_directory` | string | Yes | — | The directory work jobs will be executed from. This directory will be fully managed by the worker service and will be purged on startup. This directory and the directory referenced in `local_filesystem_store_ref`'s `stores::FilesystemStore::content_path` must be on the same filesystem. Hardlinks will be used when placing files that are accessible to the jobs that are sourced from `local_filesystem_store_ref`'s `content_path`. | +| `platform_properties` | map of string to [WorkerProperty](#workerproperty) | Yes | — | Properties of this worker. This configuration will be sent to the scheduler and used to tell the scheduler to restrict what should be executed on this worker. | +| `additional_environment` | map of string to [EnvironmentSource](#environmentsource) | — | — | An optional mapping of environment names to set for the execution as well as those specified in the action itself. If set, will set each key as an environment variable before executing the job with the value of the environment variable being the value of the property of the action being executed of that name or the fixed value. | +| `directory_cache` | [DirectoryCacheConfig](#directorycacheconfig) | — | — | Optional directory cache configuration for improving performance by caching reconstructed input directories and using hardlinks instead of rebuilding them from CAS for every action. | +| `use_namespaces` | boolean | — | False | Whether to use namespaces to isolate the execution. This is only available on Linux. It is highly recommended as it avoids a number of issues with zombie processes and also provides additional hermeticity. If explicitly set to true and it is not supported the worker will exit with an error. | +| `use_mount_namespace` | boolean | — | False | Whether to use a mount namespace to isolate the worker root. This is only available on Linux and when `use_namespaces` is true. It is highly recommended provides additional hermeticity. If explicitly set to true and it is not supported or `use_namespaces` is not set to true the worker will exit with an error. | + +## SimpleSpec + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `supported_platform_properties` | map of string to [PropertyType](#propertytype) | — | — | A list of supported platform properties mapped to how these properties are used when the scheduler looks for worker nodes capable of running the task. | +| `retain_completed_for_s` | integer (uint32) | — | 60 seconds | The amount of time to retain completed actions for in case a `WaitExecution` is called after the action has completed. | +| `client_action_timeout_s` | integer (uint64) | — | 60 seconds | Mark operations as completed with error if no client has updated them within this duration. | +| `worker_timeout_s` | integer (uint64) | — | 5 seconds | Remove workers from pool once the worker has not responded in this amount of time in seconds. | +| `max_action_executing_timeout_s` | integer (uint64) | — | 0 (disabled) | Maximum time (seconds) an action can stay in Executing state without any worker update before being timed out and re-queued. This applies regardless of worker keepalive status, catching cases where a worker is alive (sending keepalives) but stuck on a specific action. Set to 0 to disable (relies only on `worker_timeout_s`). | +| `max_job_retries` | integer (uint) | — | 3 | If a job returns an internal error or times out this many times when attempting to run on a worker the scheduler will return the last error to the client. Jobs will be retried and this configuration is to help prevent one rogue job from infinitely retrying and taking up a lot of resources when the task itself is the one causing the server to go into a bad state. | +| `allocation_strategy` | [WorkerAllocationStrategy](#workerallocationstrategy) | — | `"least_recently_used"` | The strategy used to assign workers jobs. | +| `experimental_backend` | [ExperimentalSimpleSchedulerBackend](#experimentalsimpleschedulerbackend) | — | memory | The storage backend to use for the scheduler. | +| `worker_match_logging_interval_s` | integer (int64) | — | `10` | Every N seconds, do logging of worker matching e.g. "worker busy", "can't find any worker" Defaults to 10s. Can be set to `-1` to disable | + +## SchedulerGrpcSpec + +A scheduler that forwards requests to an upstream scheduler. This +is useful to use when doing some kind of local action cache or CAS away from +the main cluster of workers. In general, it's more efficient to point the +build at the main scheduler directly though. + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `endpoint` | [GrpcEndpoint](#grpcendpoint) | Yes | — | The upstream scheduler to forward requests to. | +| `retry` | [Retry](#retry) | — | — | Retry configuration to use when a network request fails. | +| `max_concurrent_requests` | integer (uint) | — | unlimited | Limit the number of simultaneous upstream requests to this many. A value of zero is treated as unlimited. If the limit is reached the request is queued. | +| `connections_per_endpoint` | integer (uint) | — | 1 | The number of connections to make to each specified endpoint to balance the load over multiple TCP connections. | + +## CacheLookupSpec + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `ac_store` | string | Yes | — | The reference to the action cache store used to return cached actions from rather than running them again. To prevent unintended issues, this store should probably be a `CompletenessCheckingSpec`. | +| `scheduler` | [SchedulerSpec](#schedulerspec) | Yes | — | The nested scheduler to use if cache lookup fails. | + +## PropertyModifierSpec + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `modifications` | array of [PropertyModification](#propertymodification) | Yes | — | A list of modifications to perform to incoming actions for the nested scheduler. These are performed in order and blindly, so removing a property that doesn't exist is fine and overwriting an existing property is also fine. If adding properties that do not exist in the nested scheduler is not supported and will likely cause unexpected behaviour. | +| `scheduler` | [SchedulerSpec](#schedulerspec) | Yes | — | The nested scheduler to use after modifying the properties. | + +## HistoricalResourceSpec + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `hints_file` | string | Yes | — | JSON file containing historical resource hints keyed by Bazel `RequestMetadata` `target_id` and/or `action_mnemonic`. | +| `refresh_interval_s` | integer (uint64) | — | 30 seconds | Reload interval for `hints_file`. Set to 0 to load once. | +| `cpu_property_name` | string | — | `cpu_count` | Platform property name used for CPU minimum values. | +| `memory_property_name` | string | — | `memory_kb` | Platform property name used for memory minimum values, expressed in KiB. | +| `scheduler` | [SchedulerSpec](#schedulerspec) | Yes | — | The nested scheduler to use after applying resource hints. | + +## ListenerConfig + +Plus exactly one of the following variants (the key selects the variant): + +### `http` + +Listener for HTTP/HTTPS/HTTP2 sockets. + +**Type:** [HttpListener](#httplistener) + +## ServicesConfig + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `cas` | array of [CasServiceConfig](#casserviceconfig) | — | — | The Content Addressable Storage (CAS) backend config. The key is the `instance_name` used in the protocol and the value is the underlying CAS store config. | +| `ac` | array of [ActionCacheServiceConfig](#actioncacheserviceconfig) | — | — | The Action Cache (AC) backend config. The key is the `instance_name` used in the protocol and the value is the underlying AC store config. | +| `capabilities` | array of [CapabilitiesServiceConfig](#capabilitiesserviceconfig) | — | — | Capabilities service is required in order to use most of the bazel protocol. This service is used to provide the supported features and versions of this bazel GRPC service. | +| `execution` | array of [ExecutionServiceConfig](#executionserviceconfig) | — | — | The remote execution service configuration. NOTE: This service is under development and is currently just a place holder. | +| `bytestream` | array of [ByteStreamServiceConfig](#bytestreamserviceconfig) | — | — | This is the service used to stream data to and from the CAS. Bazel's protocol strongly encourages users to use this streaming interface to interact with the CAS when the data is large. | +| `fetch` | array of [FetchServiceConfig](#fetchserviceconfig) | — | — | These two are collectively the Remote Asset protocol, but it's defined as two separate services | +| `push` | array of [PushServiceConfig](#pushserviceconfig) | — | — | | +| `worker_api` | [WorkerApiConfig](#workerapiconfig) | — | — | This is the service used for workers to connect and communicate through. NOTE: This service should be served on a different, non-public port. In other words, `worker_api` configuration should not have any other services that are served on the same port. Doing so is a security risk, as workers have a different permission set than a client that makes the remote execution/cache requests. | +| `experimental_bep` | [BepConfig](#bepconfig) | — | — | Experimental - Build Event Protocol (BEP) configuration. This is the service that will consume build events from the client and publish them to a store for processing by an external service. | +| `admin` | [AdminConfig](#adminconfig) | — | — | This is the service for any administrative tasks. It provides a REST API endpoint for administrative purposes. | +| `health` | [HealthConfig](#healthconfig) | — | — | This is the service for health status check. | + +## IdentityHeaderSpec + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `header_name` | string | — | "x-identity" | The name of the header to look for the identity in. | +| `required` | boolean | — | `false` | If the header is required to be set or fail the request. | + +## OriginEventsPublisherSpec + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `store` | string | Yes | — | The store to publish nativelink events to. The store name referenced in the `stores` map in the main config. | + +## ConfigDigestHashFunction + +| Value | Description | +| --- | --- | +| `"sha256"` | Use the sha256 hash function. [https://en.wikipedia.org/wiki/SHA-2](https://en.wikipedia.org/wiki/SHA-2) | +| `"blake3"` | Use the blake3 hash function. [https://en.wikipedia.org/wiki/BLAKE_(hash_function)](https://en.wikipedia.org/wiki/BLAKE_(hash_function)) | + +## EvictionPolicy + +Eviction policy always works on LRU (Least Recently Used). Any time an entry +is touched it updates the timestamp. Inserts and updates will execute the +eviction policy removing any expired entries and/or the oldest entries +until the store size becomes smaller than `max_bytes`. + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `max_bytes` | integer (uint) | — | 0 | Maximum number of bytes before eviction takes place. Zero means never evict based on size. | +| `evict_bytes` | integer (uint) | — | 0 | When eviction starts based on hitting `max_bytes`, continue until `max_bytes - evict_bytes` is met to create a low watermark. This stops operations from thrashing when the store is close to the limit. | +| `max_seconds` | integer (uint32) | — | 0 | Maximum number of seconds for an entry to live since it was last accessed before it is evicted. Zero means never evict based on time. | +| `max_count` | integer (uint64) | — | 0 | Maximum size of the store before an eviction takes place. Zero means never evict based on count. | + +## Retry + +Retry configuration. This configuration is exponential and each iteration +a jitter as a percentage is applied of the calculated delay. For example: +```haskell +Retry{ + max_retries: 7, + delay: 0.1, + jitter: 0.5, +} +``` +will result in: + +| Attempt | Delay | +| --- | --- | +| 1 | 0 ms | +| 2 | 75 to 125 ms | +| 3 | 150 to 250 ms | +| 4 | 300 to 500 ms | +| 5 | 600 ms to 1 s | +| 6 | 1.2 to 2 s | +| 7 | 2.4 to 4 s | +| 8 | 4.8 to 8 s | + +The total delay is additive, so this example produces 9.525 to 15.875 s of total delay for a single request. + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `max_retries` | integer (uint) | — | `0` | Maximum number of retries until retrying stops. Setting this to zero will always attempt 1 time, but not retry. | +| `delay` | number (float) | — | `0` | Delay in seconds for exponential back off. | +| `jitter` | number (float) | — | `0` | Amount of jitter to add as a percentage in decimal form. This will change the formula like: | +| `retry_on_errors` | array of [ErrorCode](#errorcode) | — | — | A list of error codes to retry on, if this is not set then the default error codes to retry on are used. These default codes are the most likely to be non-permanent: `Unknown`; `Cancelled`; `DeadlineExceeded`; `ResourceExhausted`; `Aborted`; `Internal`; `Unavailable`; `DataLoss` | + +## ExperimentalOntapS3Spec + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `endpoint` | string | Yes | — | | +| `vserver_name` | string | Yes | — | | +| `bucket` | string | Yes | — | | +| `root_certificates` | string | — | — | | +| `key_prefix` | string | — | — | If you wish to prefix the location in the bucket. If None, no prefix will be used. | +| `retry` | [Retry](#retry) | — | — | Retry configuration to use when a network request fails. | +| `consider_expired_after_s` | integer (uint32) | — | 0 | If the number of seconds since the `last_modified` time of the object is greater than this value, the object will not be considered "existing". This allows for external tools to delete objects that have not been uploaded in a long time. If a client receives a `NotFound` the client should re-upload the object. | +| `max_retry_buffer_per_request` | integer (uint) | — | 5MB | The maximum buffer size to retain in case of a retryable error during upload. Setting this to zero will disable upload buffering; this means that in the event of a failure during upload, the entire upload will be aborted and the client will likely receive an error. | +| `multipart_max_concurrent_uploads` | integer (uint) | — | 10 | Maximum number of concurrent `UploadPart` requests per `MultipartUpload`. | +| `insecure_allow_http` | boolean | — | false | Allow unencrypted HTTP connections. Only use this for local testing. | +| `disable_http2` | boolean | — | false | Disable HTTP/2 connections and only use HTTP/1.1. Default client configuration will have HTTP/1.1 and HTTP/2 enabled for connection schemes. HTTP/2 should be disabled if environments have poor support or performance related to HTTP/2. Safe to keep default unless underlying network environment, S3, or GCS API servers specify otherwise. | + +## CompressionAlgorithm + +Plus exactly one of the following variants (the key selects the variant): + +### `lz4` + +LZ4 compression algorithm is extremely fast for compression and +decompression, however does not perform very well in compression +ratio. In most cases build artifacts are highly compressible, however +lz4 is quite good at aborting early if the data is not deemed very +compressible. + +see: [https://lz4.github.io/lz4/](https://lz4.github.io/lz4/) + +**Type:** [Lz4Config](#lz4config) + +## StoreDirection + +| Value | Description | +| --- | --- | +| `"both"` | The store operates normally and all get and put operations are handled by it. | +| `"update"` | Update operations will cause persistence to this store, but Get operations will be ignored. This only makes sense on the fast store as the slow store will never get written to on Get anyway. | +| `"get"` | Get operations will cause persistence to this store, but Update operations will be ignored. | +| `"read_only"` | Operate as a read only store, only really makes sense if there's another way to write to it. | + +## ShardConfig + +Configuration for an individual shard of the store. + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `store` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | Store to shard the data to. | +| `weight` | integer (uint32) | — | 1 | The weight of the store. This is used to determine how much data should be sent to the store. The actual percentage is the sum of all the store's weights divided by the individual store's weight. | + +## GrpcEndpoint + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `address` | string | Yes | — | The endpoint address (i.e. grpc://example.com:443 or grpcs://example.com:443). | +| `tls_config` | [ClientTlsConfig](#clienttlsconfig) | — | — | The TLS configuration to use to connect to the endpoint (if grpcs). | +| `concurrency_limit` | integer (uint) | — | — | The maximum concurrency to allow on this endpoint. | +| `connect_timeout_s` | integer (uint64) | — | 30 seconds | Timeout for establishing a TCP connection to the endpoint (seconds). | +| `tcp_keepalive_s` | integer (uint64) | — | 30 seconds | TCP keepalive interval (seconds). Sends TCP keepalive probes at this interval to detect dead connections at the OS level. | +| `http2_keepalive_interval_s` | integer (uint64) | — | 30 seconds | HTTP/2 keepalive interval (seconds). Sends HTTP/2 PING frames at this interval to detect dead connections at the application level. | +| `http2_keepalive_timeout_s` | integer (uint64) | — | 20 seconds | HTTP/2 keepalive timeout (seconds). If a PING response is not received within this duration, the connection is considered dead. | + +## StoreType + +| Value | Description | +| --- | --- | +| `"cas"` | The store is content addressable storage. | +| `"ac"` | The store is an action cache. | + +## RedisMode + +| Value | Description | +| --- | --- | +| `"cluster"` | Use Redis Cluster. | +| `"sentinel"` | Use Redis Sentinel. | +| `"standard"` | Use a standalone Redis server. | + +## EndpointConfig + +Generic config for an endpoint and associated configs. + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `uri` | string | Yes | — | URI of the endpoint. | +| `timeout` | number (float) | — | 5 seconds | Timeout in seconds that a request should take. | +| `tls_config` | [ClientTlsConfig](#clienttlsconfig) | — | — | The TLS configuration to use to connect to the endpoint. | + +## UploadActionResultConfig + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `ac_store` | string | — | {No uploading is done} | Underlying AC store that the worker will use to publish execution results into. Objects placed in this store should be reachable from the scheduler/client-cas after they have finished updating. | +| `upload_ac_results_strategy` | [UploadCacheResultsStrategy](#uploadcacheresultsstrategy) | — | `SuccessOnly` | In which situations should the results be published to the `ac_store`, if set to `SuccessOnly` then only results with an exit code of 0 will be uploaded, if set to Everything all completed results will be uploaded. | +| `historical_results_store` | string | — | {CAS store of parent} | Store to upload historical results to. This should be a CAS store if set. | +| `upload_historical_results_strategy` | [UploadCacheResultsStrategy](#uploadcacheresultsstrategy) | — | `FailuresOnly` | In which situations should the results be published to the historical CAS. The historical CAS is where failures are published. These messages conform to the CAS key-value lookup format and are always a `HistoricalExecuteResponse` serialized message. | +| `success_message_template` | string | — | "" (no message) | Template to use for the `ExecuteResponse.message` property. This message is attached to the response before it is sent to the client. The following special variables are supported:; `digest_function`: Digest function used to calculate the action digest: `action_digest_hash`: Action digest hash: `action_digest_size`: Action digest size: `historical_results_hash`: `HistoricalExecuteResponse` digest hash: `historical_results_size`: `HistoricalExecuteResponse` digest size. | +| `failure_message_template` | string | — | "" (no message) | Same as `success_message_template` but for failure case. | + +## WorkerProperty + +Plus exactly one of the following variants (the key selects the variant): + +### `values` + +List of static values. +Note: Generally there should only ever be 1 value, but if the platform +property key is `PropertyType::Priority` it may have more than one value. + +**Type:** array of string + +### `query_cmd` + +A dynamic configuration. The string will be executed as a command +(not shell) and will be split by "\n" (new line character). + +**Type:** string + +## EnvironmentSource + +One of: + +- `property`: string — The name of the platform property in the action to get the value from. +- `value`: string — The raw value to set. +- `"from_environment"` — Take the value from the local environment corresponding to the name key +- `"timeout_millis"` — The max amount of time in milliseconds the command is allowed to run (requested by the client). +- `"side_channel_file"` — A special file path will be provided that can be used to communicate with the parent process about out-of-band information. This file will be read after the command has finished executing. Based on the contents of the file, the behavior of the result may be modified. +- `"action_directory"` — A "root" directory for the action. This directory can be used to store temporary files that are not needed after the action has completed. This directory will be purged after the action has completed. + +## DirectoryCacheConfig + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `max_entries` | integer (uint) | — | 1000 | Maximum number of cached directories. | +| `max_size_bytes` | integer (uint64) | — | 10737418240 (10 GB) | Maximum total size in bytes for all cached directories (0 = unlimited). | +| `cache_root` | string | — | `{work_directory}/../directory_cache` | Base directory for cache storage. This directory will be managed by the worker and should be on the same filesystem as `work_directory`. | + +## PropertyType + +When the scheduler matches tasks to workers that are capable of running +the task, this value will be used to determine how the property is treated. + +One of: + +- string +- `"minimum"` — Requires the platform property to be a u64 and when the scheduler looks for appropriate worker nodes that are capable of executing the task, the task will not run on a node that has less than this value. +- `"exact"` — Requires the platform property to be a string and when the scheduler looks for appropriate worker nodes that are capable of executing the task, the task will not run on a node that does not have this property set to the value with exact string match. +- `"priority"` — Does not restrict on this value and instead will be passed to the worker as an informational piece. In the future this will be used by the scheduler and worker to cause the scheduler to prefer certain workers over others, but not restrict them based on these values. + +## WorkerAllocationStrategy + +When a worker is being searched for to run a job, this will be used +on how to choose which worker should run the job when multiple +workers are able to run the task. + +| Value | Description | +| --- | --- | +| `"least_recently_used"` | Prefer workers that have been least recently used to run a job. | +| `"most_recently_used"` | Prefer workers that have been most recently used to run a job. | + +## ExperimentalSimpleSchedulerBackend + +One of: + +- `"memory"` — Use an in-memory store for the scheduler. +- `redis`: [ExperimentalRedisSchedulerBackend](#experimentalredisschedulerbackend) — Use a Redis store for the scheduler. + +## SchedulerSpec + +Plus exactly one of the following variants (the key selects the variant): + +### `simple` + +**Type:** [SimpleSpec](#simplespec) + +### `grpc` + +**Type:** [SchedulerGrpcSpec](#schedulergrpcspec) + +### `cache_lookup` + +**Type:** [CacheLookupSpec](#cachelookupspec) + +### `property_modifier` + +**Type:** [PropertyModifierSpec](#propertymodifierspec) + +### `historical_resource` + +**Type:** [HistoricalResourceSpec](#historicalresourcespec) + +## PropertyModification + +Plus exactly one of the following variants (the key selects the variant): + +### `add` + +Add a property to the action properties. + +**Type:** [PlatformPropertyAddition](#platformpropertyaddition) + +### `remove` + +Remove a named property from the action. + +**Type:** string + +### `replace` + +If a property is found, then replace it with another one. + +**Type:** [PlatformPropertyReplacement](#platformpropertyreplacement) + +## HttpListener + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `socket_address` | string | Yes | — | Address to listen on. Example: `127.0.0.1:8080` or `:8080` to listen to all IPs. | +| `freebind` | boolean | — | false | Allow binding `socket_address` before it is assigned locally. | +| `compression` | [HttpCompressionConfig](#httpcompressionconfig) | — | — | Data transport compression configuration to use for this service. | +| `advanced_http` | [HttpServerConfig](#httpserverconfig) | — | — | Advanced HTTP server configuration. | +| `max_decoding_message_size` | integer (uint) | — | 4 MiB | Maximum number of bytes to decode on each grpc stream chunk. | +| `tls` | [TlsConfig](#tlsconfig) | — | — | TLS configuration for this server. If not set, the server will not use TLS. | + +## CasServiceConfig + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `instance_name` | string | — | `""` | Used when the config references `instance_name` in the protocol. | +| `cas_store` | string | Yes | — | The store name referenced in the `stores` map in the main config. This store name referenced here may be reused multiple times. | + +## ActionCacheServiceConfig + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `instance_name` | string | — | `""` | Used when the config references `instance_name` in the protocol. | +| `ac_store` | string | Yes | — | The store name referenced in the `stores` map in the main config. This store name referenced here may be reused multiple times. | +| `read_only` | boolean | — | `false` | Whether the Action Cache store may be written to, this if set to false it is only possible to read from the Action Cache. | + +## CapabilitiesServiceConfig + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `instance_name` | string | — | `""` | Used when the config references `instance_name` in the protocol. | +| `remote_execution` | [CapabilitiesRemoteExecutionConfig](#capabilitiesremoteexecutionconfig) | — | — | Configuration for remote execution capabilities. If not set the capabilities service will inform the client that remote execution is not supported. | + +## ExecutionServiceConfig + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `instance_name` | string | — | `""` | Used when the config references `instance_name` in the protocol. | +| `cas_store` | string | Yes | — | The store name referenced in the `stores` map in the main config. This store name referenced here may be reused multiple times. This value must be a CAS store reference. | +| `scheduler` | string | Yes | — | The scheduler name referenced in the `schedulers` map in the main config. | + +## ByteStreamServiceConfig + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `instance_name` | string | — | `""` | Used when the config references `instance_name` in the protocol. | +| `cas_store` | string | Yes | — | Name of the store in the "stores" configuration. | +| `max_bytes_per_stream` | integer (uint) | — | 64KiB | Max number of bytes to send on each grpc stream chunk. According to [https://github.com/grpc/grpc.github.io/issues/371](https://github.com/grpc/grpc.github.io/issues/371) 16KiB - 64KiB is optimal. | +| `persist_stream_on_disconnect_timeout` | integer (uint) | — | 10 seconds | In the event a client disconnects while uploading a blob, we will hold the internal stream open for this many seconds before closing it. This allows clients that disconnect to reconnect and continue uploading the same blob. | + +## FetchServiceConfig + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `instance_name` | string | — | `""` | Used when the config references `instance_name` in the protocol. | +| `fetch_store` | string | Yes | — | The store name referenced in the `stores` map in the main config. This store name referenced here may be reused multiple times. | + +## PushServiceConfig + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `instance_name` | string | — | `""` | Used when the config references `instance_name` in the protocol. | +| `push_store` | string | Yes | — | The store name referenced in the `stores` map in the main config. This store name referenced here may be reused multiple times. | +| `read_only` | boolean | — | `false` | Whether the Action Cache store may be written to, this if set to false it is only possible to read from the Action Cache. | + +## WorkerApiConfig + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `scheduler` | string | Yes | — | The scheduler name referenced in the `schedulers` map in the main config. | + +## BepConfig + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `store` | string | Yes | — | The store to publish build events to. The store name referenced in the `stores` map in the main config. | + +## AdminConfig + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `path` | string | — | "/admin" | Path to register the admin API. If path is "/admin", and your domain is "example.com", you can reach the endpoint with: [http://example.com/admin](http://example.com/admin). | + +## HealthConfig + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `path` | string | — | "/status" | Path to register the health status check. If path is "/status", and your domain is "example.com", you can reach the endpoint with: [http://example.com/status](http://example.com/status). | +| `timeout_seconds` | integer (uint64) | — | 5s | Timeout on health checks. | + +## ErrorCode + +The possible error codes that might occur on an upstream request. + +| Value | Description | +| --- | --- | +| `"Cancelled"` | | +| `"Unknown"` | | +| `"InvalidArgument"` | | +| `"DeadlineExceeded"` | | +| `"NotFound"` | | +| `"AlreadyExists"` | | +| `"PermissionDenied"` | | +| `"ResourceExhausted"` | | +| `"FailedPrecondition"` | | +| `"Aborted"` | | +| `"OutOfRange"` | | +| `"Unimplemented"` | | +| `"Internal"` | | +| `"Unavailable"` | | +| `"DataLoss"` | | +| `"Unauthenticated"` | | + +## Lz4Config + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `block_size` | integer (uint32) | — | 65536 (64k) | Size of the blocks to compress. Higher values require more ram, but might yield slightly better compression ratios. | +| `max_decode_block_size` | integer (uint32) | — | value in `block_size` | Maximum size allowed to attempt to deserialize data into. This is needed because the `block_size` is embedded into the data so if there was a bad actor, they could upload an extremely large `block_size`'ed entry and we'd allocate a large amount of memory when retrieving the data. To prevent this from happening, we allow you to specify the maximum that we'll attempt to deserialize. | + +## ClientTlsConfig + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `ca_file` | string | — | — | Path to the certificate authority to use to validate the remote. | +| `cert_file` | string | — | — | Path to the certificate file for client authentication. | +| `key_file` | string | — | — | Path to the private key file for client authentication. | +| `use_native_roots` | boolean | — | false | If set the client will use the native roots for TLS connections. | + +## UploadCacheResultsStrategy + +| Value | Description | +| --- | --- | +| `"success_only"` | Only upload action results with an exit code of 0. | +| `"never"` | Don't upload any action results. | +| `"everything"` | Upload all action results that complete. | +| `"failures_only"` | Only upload action results that fail. | + +## ExperimentalRedisSchedulerBackend + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `redis_store` | string | Yes | — | A reference to the Redis store to use for the scheduler. Note: This MUST resolve to a `RedisSpec`. | + +## PlatformPropertyAddition + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `name` | string | Yes | — | The name of the property to add. | +| `value` | string | Yes | — | The value to assign to the property. | + +## PlatformPropertyReplacement + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `name` | string | Yes | — | The name of the property to replace. | +| `value` | string | — | — | The value to match against, if unset then any instance matches. | +| `new_name` | string | Yes | — | The new name of the property. | +| `new_value` | string | — | — | The value to assign to the property, if unset will remain the same. | + +## HttpCompressionConfig + +Note: Compressing data in the cloud rarely has a benefit, since most +cloud providers have very high bandwidth backplanes. However, for +clients not inside the data center, it might be a good idea to +compress data to and from the cloud. This will however come at a high +CPU and performance cost. If you are making remote execution share the +same CAS/AC servers as client's remote cache, you can create multiple +services with different compression settings that are served on +different ports. Then configure the non-cloud clients to use one port +and cloud-clients to use another. + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `send_compression_algorithm` | [HttpCompressionAlgorithm](#httpcompressionalgorithm) | — | `HttpCompressionAlgorithm::None` | The compression algorithm that the server will use when sending responses to clients. Enabling this will likely save a lot of data transfer, but will consume a lot of CPU and add a lot of latency. see: [https://github.com/tracemachina/nativelink/issues/109](https://github.com/tracemachina/nativelink/issues/109) | +| `accepted_compression_algorithms` | array of [HttpCompressionAlgorithm](#httpcompressionalgorithm) | Yes | {no supported compression} | The compression algorithm that the server will accept from clients. The server will broadcast the supported compression algorithms to clients and the client will choose which compression algorithm to use. Enabling this will likely save a lot of data transfer, but will consume a lot of CPU and add a lot of latency. see: [https://github.com/tracemachina/nativelink/issues/109](https://github.com/tracemachina/nativelink/issues/109) | + +## HttpServerConfig + +Advanced HTTP configuration. These generally should not be set. +For documentation on these settings, see the hyper documentation: +See: [hyper HTTP server docs](https://docs.rs/hyper/latest/hyper/server/conn/struct.Http.html) + +Note: All of these default to the default values from hyper unless otherwise +specified. + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `http2_keep_alive_interval` | integer (uint32) | — | — | Interval to send keep-alive pings via HTTP2. Note: This is in seconds. | +| `experimental_http2_max_pending_accept_reset_streams` | integer (uint32) | — | — | | +| `experimental_http2_initial_stream_window_size` | integer (uint32) | — | — | | +| `experimental_http2_initial_connection_window_size` | integer (uint32) | — | — | | +| `experimental_http2_adaptive_window` | boolean | — | — | | +| `experimental_http2_max_frame_size` | integer (uint32) | — | — | | +| `experimental_http2_max_concurrent_streams` | integer (uint32) | — | — | | +| `experimental_http2_keep_alive_timeout` | integer (uint32) | — | — | Note: This is in seconds. | +| `experimental_http2_max_send_buf_size` | integer (uint32) | — | — | | +| `experimental_http2_enable_connect_protocol` | boolean | — | — | | +| `experimental_http2_max_header_list_size` | integer (uint32) | — | — | | + +## TlsConfig + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `cert_file` | string | Yes | — | Path to the certificate file. | +| `key_file` | string | Yes | — | Path to the private key file. | +| `client_ca_file` | string | — | — | Path to the certificate authority for mTLS, if client authentication is required for this endpoint. | +| `client_crl_file` | string | — | — | Path to the certificate revocation list for mTLS, if client authentication is required for this endpoint. | + +## CapabilitiesRemoteExecutionConfig + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `scheduler` | string | Yes | — | Scheduler used to configure the capabilities of remote execution. | + +## HttpCompressionAlgorithm + +| Value | Description | +| --- | --- | +| `"none"` | No compression. | +| `"gzip"` | Zlib compression. | + +## Reading the source + +If anything here disagrees with the binary, the source wins: + +- [`stores.rs`](https://github.com/TraceMachina/nativelink/tree/v1.5.2/nativelink-config/src/stores.rs) +- [`cas_server.rs`](https://github.com/TraceMachina/nativelink/tree/v1.5.2/nativelink-config/src/cas_server.rs) +- [`schedulers.rs`](https://github.com/TraceMachina/nativelink/tree/v1.5.2/nativelink-config/src/schedulers.rs) diff --git a/web/apps/docs/lib/config-versions.ts b/web/apps/docs/lib/config-versions.ts index 624a044b4..5ba51fd01 100644 --- a/web/apps/docs/lib/config-versions.ts +++ b/web/apps/docs/lib/config-versions.ts @@ -28,13 +28,29 @@ export const CONFIG_VERSIONS: ConfigVersion[] = [ "isDev": true }, { - "version": "v1.5.2", - "label": "v1.5.2 (latest)", + "version": "v1.6.1", + "label": "v1.6.1 (latest)", "href": "/reference/nativelink-config", - "ref": "v1.5.2", + "ref": "v1.6.1", "isLatest": true, "isDev": false }, + { + "version": "v1.6.0", + "label": "v1.6.0", + "href": "/reference/nativelink-config/v1.6.0", + "ref": "v1.6.0", + "isLatest": false, + "isDev": false + }, + { + "version": "v1.5.2", + "label": "v1.5.2", + "href": "/reference/nativelink-config/v1.5.2", + "ref": "v1.5.2", + "isLatest": false, + "isDev": false + }, { "version": "v1.5.1", "label": "v1.5.1", @@ -99,14 +115,6 @@ export const CONFIG_VERSIONS: ConfigVersion[] = [ "isLatest": false, "isDev": false }, - { - "version": "v1.0.1", - "label": "v1.0.1", - "href": "/reference/nativelink-config/v1.0.1", - "ref": "v1.0.1", - "isLatest": false, - "isDev": false - }, { "version": "v1.0.0", "label": "v1.0.0", From 4be35e6dccb7fbc08d414ca029c592e9e7853cbf Mon Sep 17 00:00:00 2001 From: Ernesto Cambuston Date: Thu, 16 Jul 2026 05:11:09 -0700 Subject: [PATCH 05/84] Add opt-in small-blob read coalescing to GrpcStore (#2540) When `experimental_read_batching` is configured on a gRPC store, full reads of small blobs are coalesced into BatchReadBlobs RPCs instead of issuing one ByteStream Read stream per blob. Each ByteStream read carries a ~1.6ms fixed per-RPC cost; batched reads measured 52us/blob at 4KiB (30.2x), 4.8x at 32KiB and 1.8x at 256KiB against a real in-process gRPC server. The coalescer uses slot-based group commit with no timers: callers enqueue their read and try_acquire one of `dispatch_slots` semaphore permits to become a dispatcher. A dispatcher drains up to `max_batch_bytes` of pending reads (grouped by digest function, with a per-entry overhead charge to bound entry counts) into a single BatchReadBlobs request, deduplicates digests and fans response data out to every waiter of a digest, validates entry size and identity compression, and keeps draining until the queue is empty. After releasing its permit the dispatcher re-checks the queue to close the race with concurrent enqueues. When more than `max_queued_bytes` are already waiting, new reads bypass batching and use the stream path so the coalescer never blocks. Per-item failures (e.g. NOT_FOUND) fail only that read; retryable per-item errors fall back to the ByteStream path, which re-enters the retry machinery; whole-RPC failures are broadcast to the batch. Dispatchers run as detached background tasks holding a strong reference obtained via a weak self pointer: cancellation of any individual reader (timeouts, try_join siblings) can neither abort an in-flight batch RPC nor strand still-queued waiters - a cancelled caller only drops its own result receiver. Because batched reads share one upstream RPC across many client requests, `experimental_read_batching` is rejected at construction when combined with `forward_headers`, whose per-client values (e.g. credentials) cannot be attached to a shared RPC. The coalescer exports metrics under the store's read_batcher group: batches_sent, blobs_batched, queue_bypasses, batched_read_errors and a queued_bytes gauge (atomic mirror of the queue byte budget). Partial reads, blobs above `max_blob_size_bytes`, non-digest keys, AC stores and empty blobs keep the existing behavior. The feature is off by default and unset config is zero behavior change. Co-authored-by: Claude Fable 5 Co-authored-by: Marcus Eagan --- .../vocabularies/TraceMachina/accept.txt | 1 + nativelink-config/src/stores.rs | 84 +++ nativelink-service/tests/cas_server_test.rs | 1 + nativelink-store/BUILD.bazel | 1 + nativelink-store/src/grpc_store.rs | 373 +++++++++++- .../tests/grpc_read_batching_test.rs | 572 ++++++++++++++++++ nativelink-store/tests/grpc_store_test.rs | 1 + 7 files changed, 1027 insertions(+), 6 deletions(-) create mode 100644 nativelink-store/tests/grpc_read_batching_test.rs diff --git a/.github/styles/config/vocabularies/TraceMachina/accept.txt b/.github/styles/config/vocabularies/TraceMachina/accept.txt index 4725cfdb8..24af1afe0 100644 --- a/.github/styles/config/vocabularies/TraceMachina/accept.txt +++ b/.github/styles/config/vocabularies/TraceMachina/accept.txt @@ -9,6 +9,7 @@ Bazelisk [Kk]eyless Sigstore Cloudflare +[Cc]oalescers? Colab composable CPUs diff --git a/nativelink-config/src/stores.rs b/nativelink-config/src/stores.rs index d811b01a2..c97c1fceb 100644 --- a/nativelink-config/src/stores.rs +++ b/nativelink-config/src/stores.rs @@ -1440,6 +1440,90 @@ pub struct GrpcSpec { /// context (`traceparent` / `tracestate`) into every outgoing request. #[serde(default)] pub forward_headers: Vec, + + /// Optional and experimental: coalesce small-blob reads into + /// `BatchReadBlobs` RPCs instead of issuing one `ByteStream` `Read` + /// stream per blob. Each `ByteStream` read carries a fixed per-RPC cost, + /// so batching many small reads into a single `BatchReadBlobs` request + /// can dramatically reduce read latency for small blobs. + /// + /// Only full reads (offset 0, whole blob) of blobs at or below + /// `max_blob_size_bytes` are batched; everything else continues to use + /// the `ByteStream` `Read` path. + /// + /// Incompatible with `forward_headers`: batched reads share one upstream + /// RPC across many client requests, so per-client forwarded headers + /// (e.g. credentials) cannot be attached. Configuring both is rejected + /// at startup. + /// + /// Default: unset (disabled). When unset there is zero behavior change. + #[serde(default)] + pub experimental_read_batching: Option, +} + +/// Configuration for experimental small-blob read coalescing in a gRPC +/// store. See [`GrpcSpec::experimental_read_batching`]. +#[derive(Serialize, Deserialize, Debug, Clone, Copy)] +#[serde(deny_unknown_fields)] +#[cfg_attr(feature = "dev-schema", derive(JsonSchema))] +pub struct GrpcReadBatchingConfig { + /// Only blobs at or below this size (in bytes) are eligible for + /// batching. Larger blobs always use the `ByteStream` `Read` path. + /// + /// Default: 131072 (128 KiB). + #[serde( + default = "default_read_batching_max_blob_size_bytes", + deserialize_with = "convert_data_size_with_shellexpand" + )] + pub max_blob_size_bytes: u64, + + /// Maximum total payload bytes packed into a single `BatchReadBlobs` + /// request. This should leave headroom under the 4 MiB default gRPC + /// message limit for protobuf framing overhead. + /// + /// Default: 3145728 (3 MiB). + #[serde( + default = "default_read_batching_max_batch_bytes", + deserialize_with = "convert_data_size_with_shellexpand" + )] + pub max_batch_bytes: u64, + + /// Maximum number of concurrent `BatchReadBlobs` RPCs dispatched by the + /// coalescer. Must be greater than zero. + /// + /// Default: 4. + #[serde( + default = "default_read_batching_dispatch_slots", + deserialize_with = "convert_numeric_with_shellexpand" + )] + pub dispatch_slots: usize, + + /// Bound on the number of payload bytes waiting in the coalescer queue. + /// When exceeded, new read requests bypass batching and fall back to the + /// regular `ByteStream` `Read` path instead of blocking. + /// + /// Default: 33554432 (32 MiB). + #[serde( + default = "default_read_batching_max_queued_bytes", + deserialize_with = "convert_data_size_with_shellexpand" + )] + pub max_queued_bytes: u64, +} + +const fn default_read_batching_max_blob_size_bytes() -> u64 { + 128 * 1024 // 128 KiB. +} + +const fn default_read_batching_max_batch_bytes() -> u64 { + 3 * 1024 * 1024 // 3 MiB. +} + +const fn default_read_batching_dispatch_slots() -> usize { + 4 +} + +const fn default_read_batching_max_queued_bytes() -> u64 { + 32 * 1024 * 1024 // 32 MiB. } /// The possible error codes that might occur on an upstream request. diff --git a/nativelink-service/tests/cas_server_test.rs b/nativelink-service/tests/cas_server_test.rs index f61095d92..32b49fd54 100644 --- a/nativelink-service/tests/cas_server_test.rs +++ b/nativelink-service/tests/cas_server_test.rs @@ -1564,6 +1564,7 @@ async fn chunking_on_grpc_store_forbids_index_store() -> Result<(), Box( request } +/// Estimated per-entry protobuf and framing overhead charged against +/// `max_batch_bytes`, so that batches of many tiny blobs cannot push a +/// `BatchReadBlobs` response over the gRPC message size limit. +const BATCH_READ_PER_ENTRY_OVERHEAD_BYTES: u64 = 256; + +/// A small-blob read waiting to be coalesced into a `BatchReadBlobs` RPC. +#[derive(Debug)] +struct PendingRead { + digest: DigestInfo, + digest_function: i32, + tx: oneshot::Sender>, +} + +/// The pending small-blob reads plus the total payload bytes they declare. +#[derive(Debug, Default)] +struct ReadQueue { + items: VecDeque, + bytes: u64, +} + +/// State for coalescing small-blob reads into `BatchReadBlobs` RPCs. +/// +/// This uses a slot-based group commit scheme: callers enqueue their read and +/// then try to start a detached dispatcher task by acquiring one of +/// `dispatch_slots` semaphore permits. A dispatcher repeatedly drains up to +/// `max_batch_bytes` worth of pending reads into a single `BatchReadBlobs` +/// request until the queue is empty. This is work-conserving (a read never +/// waits while a dispatch slot is free) and uses no timers. +#[derive(Debug, MetricsComponent)] +struct ReadBatcher { + max_blob_size_bytes: u64, + max_batch_bytes: u64, + max_queued_bytes: u64, + queue: Mutex, + dispatch_slots: Arc, + #[metric(help = "Number of BatchReadBlobs RPCs sent by the read coalescer")] + batches_sent: AtomicU64, + #[metric(help = "Number of blob reads coalesced into BatchReadBlobs RPCs")] + blobs_batched: AtomicU64, + #[metric( + help = "Number of reads that bypassed batching because the queue byte budget was full" + )] + queue_bypasses: AtomicU64, + #[metric(help = "Number of batched reads that resolved to a per-entry error")] + batched_read_errors: AtomicU64, + #[metric(help = "Payload bytes currently waiting in the read coalescer queue")] + queued_bytes: AtomicU64, +} + +impl ReadBatcher { + fn new(config: &GrpcReadBatchingConfig) -> Self { + Self { + max_blob_size_bytes: config.max_blob_size_bytes, + max_batch_bytes: config.max_batch_bytes, + max_queued_bytes: config.max_queued_bytes, + queue: Mutex::new(ReadQueue::default()), + dispatch_slots: Arc::new(Semaphore::new(config.dispatch_slots)), + batches_sent: AtomicU64::new(0), + blobs_batched: AtomicU64::new(0), + queue_bypasses: AtomicU64::new(0), + batched_read_errors: AtomicU64::new(0), + queued_bytes: AtomicU64::new(0), + } + } +} + +/// Mirrors the default retryable-code classification used by +/// `nativelink_util::retry::Retrier::should_retry`: only codes that are +/// always terminal are considered non-retryable. +const fn is_retryable_code(code: Code) -> bool { + !matches!( + code, + Code::Ok + | Code::InvalidArgument + | Code::FailedPrecondition + | Code::OutOfRange + | Code::Unimplemented + | Code::NotFound + | Code::AlreadyExists + | Code::PermissionDenied + | Code::Unauthenticated + ) +} + // This store is usually a pass-through store, but can also be used as a CAS store. Using it as an // AC store has one major side-effect... The has() function may not give the proper size of the // underlying data. This might cause issues if embedded in certain stores. @@ -120,6 +207,13 @@ pub struct GrpcStore { use_legacy_resource_names: bool, headers: Vec<(MetadataKey, MetadataValue)>, forward_headers: Vec, + /// When configured, coalesces small-blob reads into `BatchReadBlobs` + /// RPCs. `None` means reads always use the `ByteStream` `Read` path. + #[metric(group = "read_batcher")] + read_batcher: Option, + /// Used by the read coalescer to hand a strong reference of this store + /// to detached dispatcher tasks. + weak_self: Weak, } impl GrpcStore { @@ -146,6 +240,24 @@ impl GrpcStore { let rpc_timeout = Duration::from_secs(spec.rpc_timeout_s); + let read_batcher = match &spec.experimental_read_batching { + Some(config) => { + error_if!( + config.dispatch_slots == 0, + "experimental_read_batching.dispatch_slots must be greater than zero" + ); + // Batched reads share one upstream RPC across many client + // requests, so per-client forwarded headers (e.g. credentials) + // cannot be attached correctly. + error_if!( + !spec.forward_headers.is_empty(), + "experimental_read_batching is incompatible with forward_headers" + ); + Some(ReadBatcher::new(config)) + } + None => None, + }; + let mut headers = Vec::with_capacity(spec.headers.len()); for (name, value) in &spec.headers { // We lowercase keys as HTTP headers are case-insensitive so we should match all cases @@ -161,7 +273,8 @@ impl GrpcStore { headers.push((key, val)); } - Ok(Arc::new(Self { + Ok(Arc::new_cyclic(|weak_self| Self { + weak_self: weak_self.clone(), instance_name: spec.instance_name.clone(), store_type: spec.store_type, retrier: Retrier::new( @@ -178,6 +291,7 @@ impl GrpcStore { ), rpc_timeout, use_legacy_resource_names: spec.use_legacy_resource_names, + read_batcher, headers, // We lowercase keys as HTTP headers are case-insensitive so we should match all cases forward_headers: spec @@ -307,6 +421,217 @@ impl GrpcStore { .await } + /// Enqueues a small-blob read for coalescing into a `BatchReadBlobs` RPC + /// and waits for its result. Returns `None` when the queue is over its + /// byte budget, in which case the caller must fall back to the + /// `ByteStream` `Read` path. + async fn batched_read( + &self, + batcher: &ReadBatcher, + digest: DigestInfo, + ) -> Option> { + // Capture the digest function from the caller's ambient context now; + // the dispatcher runs on a detached task with no such context. + let digest_function: i32 = Context::current() + .get::() + .map_or_else(default_digest_hasher_func, |v| *v) + .proto_digest_func() + .into(); + let (tx, rx) = oneshot::channel(); + { + let mut queue = batcher.queue.lock(); + // Admission control. On overflow gracefully degrade to the + // stream path instead of blocking. + let new_bytes = queue.bytes.saturating_add(digest.size_bytes()); + if new_bytes > batcher.max_queued_bytes { + batcher.queue_bypasses.fetch_add(1, Ordering::Relaxed); + return None; + } + queue.bytes = new_bytes; + batcher.queued_bytes.store(new_bytes, Ordering::Relaxed); + queue.items.push_back(PendingRead { + digest, + digest_function, + tx, + }); + } + + self.maybe_dispatch_read_batches(batcher); + + Some(rx.await.unwrap_or_else(|_| { + Err(make_err!( + Code::Internal, + "Read batch dispatcher dropped result in GrpcStore::batched_read" + )) + })) + } + + /// Tries to start a read batch dispatcher. This is work-conserving: if a + /// dispatch slot is free a dispatcher is started immediately, otherwise + /// one of the active dispatchers is responsible for every currently + /// queued item. The dispatcher runs as a detached task so that + /// cancellation of any individual reader can neither abort an in-flight + /// `BatchReadBlobs` RPC nor strand still-queued waiters. + fn maybe_dispatch_read_batches(&self, batcher: &ReadBatcher) { + let Ok(mut permit) = batcher.dispatch_slots.clone().try_acquire_owned() else { + return; + }; + let Some(store) = self.weak_self.upgrade() else { + return; + }; + background_spawn!("grpc_store_read_batch_dispatch", async move { + let Some(batcher) = &store.read_batcher else { + return; + }; + loop { + store.dispatch_read_batches(batcher).await; + drop(permit); + // Items may have been enqueued between the last drain and + // the permit release. Re-check so they are not stranded + // with no active dispatcher. + if batcher.queue.lock().items.is_empty() { + return; + } + match batcher.dispatch_slots.clone().try_acquire_owned() { + Ok(new_permit) => permit = new_permit, + // Another dispatcher is active and will observe these + // items (or re-check after releasing its own permit). + Err(_) => return, + } + } + }); + } + + /// Drains the pending read queue, sending one `BatchReadBlobs` RPC per + /// drained batch, until the queue is empty. + async fn dispatch_read_batches(&self, batcher: &ReadBatcher) { + loop { + let batch = { + let mut queue = batcher.queue.lock(); + let Some(head) = queue.items.front() else { + return; + }; + // All digests in one BatchReadBlobsRequest must use the same + // digest function. Partition-drain: take items matching the + // head's digest function from anywhere in the queue (up to + // the batch budget) and keep the rest in relative order. + let digest_function = head.digest_function; + let mut batch = Vec::new(); + let mut batch_bytes = 0u64; + let mut rest = VecDeque::with_capacity(queue.items.len()); + while let Some(item) = queue.items.pop_front() { + let item_cost = item + .digest + .size_bytes() + .saturating_add(BATCH_READ_PER_ENTRY_OVERHEAD_BYTES); + if item.digest_function == digest_function + && (batch.is_empty() + || batch_bytes.saturating_add(item_cost) <= batcher.max_batch_bytes) + { + batch_bytes = batch_bytes.saturating_add(item_cost); + queue.bytes = queue.bytes.saturating_sub(item.digest.size_bytes()); + batch.push(item); + } else { + rest.push_back(item); + } + } + queue.items = rest; + batcher.queued_bytes.store(queue.bytes, Ordering::Relaxed); + batch + }; + self.send_read_batch(batcher, batch).await; + } + } + + /// Sends one `BatchReadBlobs` RPC for `batch` and demultiplexes the + /// per-blob responses back to the waiting readers. One failed item does + /// not affect its batch-mates; failure of the whole RPC is broadcast to + /// every item in the batch. + async fn send_read_batch(&self, batcher: &ReadBatcher, batch: Vec) { + let Some(digest_function) = batch.first().map(|item| item.digest_function) else { + return; + }; + // Servers may dedupe duplicate digests within one request, so group + // the waiters per digest and request each digest exactly once, + // fanning the (refcounted) data out to every waiter. + let batch_len = u64::try_from(batch.len()).unwrap_or(u64::MAX); + let mut waiters: HashMap> = HashMap::new(); + for item in batch { + waiters.entry(item.digest).or_default().push(item); + } + let request = BatchReadBlobsRequest { + // batch_read_blobs() overwrites the instance name, so there is + // no need to set it here. + instance_name: String::new(), + digests: waiters.keys().map(|digest| (*digest).into()).collect(), + acceptable_compressors: vec![], + digest_function, + }; + batcher.batches_sent.fetch_add(1, Ordering::Relaxed); + batcher + .blobs_batched + .fetch_add(batch_len, Ordering::Relaxed); + let response = match self.batch_read_blobs(Request::new(request)).await { + Ok(response) => response.into_inner(), + Err(err) => { + // The whole RPC failed, so every waiter in this batch gets + // the error. Waiters may have gone away, ignore send errors. + for item in waiters.into_values().flatten() { + drop(item.tx.send(Err(err.clone()))); + } + return; + } + }; + for entry in response.responses { + let Some(Ok(entry_digest)) = entry.digest.map(DigestInfo::try_from) else { + continue; + }; + let Some(items) = waiters.remove(&entry_digest) else { + continue; + }; + let entry_len = u64::try_from(entry.data.len()).unwrap_or(u64::MAX); + let result = if let Some(status) = entry.status.filter(|status| status.code != 0) { + Err(Error::from(status) + .append("Batch read entry failed in GrpcStore::send_read_batch")) + } else if entry.compressor != 0 { + // We requested no acceptable compressors, so data must be + // returned with the identity compressor. + Err(make_err!( + Code::Internal, + "BatchReadBlobs entry for {entry_digest} used unsupported compressor {}", + entry.compressor + )) + } else if entry_len != entry_digest.size_bytes() { + Err(make_err!( + Code::Internal, + "BatchReadBlobs entry for {entry_digest} returned {entry_len} bytes, expected {}", + entry_digest.size_bytes() + )) + } else { + Ok(entry.data) + }; + if result.is_err() { + batcher.batched_read_errors.fetch_add( + u64::try_from(items.len()).unwrap_or(u64::MAX), + Ordering::Relaxed, + ); + } + for item in items { + drop(item.tx.send(result.clone())); + } + } + // Any waiter with no matching response entry is missing upstream. + for item in waiters.into_values().flatten() { + batcher.batched_read_errors.fetch_add(1, Ordering::Relaxed); + let err = make_err!( + Code::NotFound, + "Blob {} not found in BatchReadBlobs response", + item.digest + ); + drop(item.tx.send(Err(err))); + } + } + pub async fn get_tree( &self, grpc_request: Request, @@ -946,6 +1271,7 @@ impl StoreDriver for GrpcStore { read_limit: i64, } + let is_digest_key = matches!(key, StoreKey::Digest(_)); let digest = key.into_digest(); if matches!(self.store_type, nativelink_config::stores::StoreType::Ac) { let offset = usize::try_from(offset).err_tip(|| "Could not convert offset to usize")?; @@ -963,6 +1289,41 @@ impl StoreDriver for GrpcStore { return writer.send_eof(); } + // When configured, coalesce full reads of small blobs into + // BatchReadBlobs RPCs. `batched_read` returns `None` when the queue + // is over budget, in which case we fall through to the stream path. + if let Some(batcher) = &self.read_batcher + && is_digest_key + && offset == 0 + && length.is_none_or(|len| len >= digest.size_bytes()) + && digest.size_bytes() <= batcher.max_blob_size_bytes + && let Some(result) = self.batched_read(batcher, digest).await + { + match result { + Ok(data) => { + if !data.is_empty() { + writer + .send(data) + .await + .err_tip(|| "Failed to write data in GrpcStore::get_part()")?; + } + return writer + .send_eof() + .err_tip(|| "Failed to send EOF in GrpcStore::get_part()"); + } + // A retryable error falls through to the ByteStream path + // below, which re-enters the full retry machinery. This + // matches the retry behavior reads had before batching. + Err(err) if is_retryable_code(err.code) => { + warn!( + ?err, + "Batched read failed with retryable error, falling back to ByteStream read", + ); + } + Err(err) => return Err(err.append("in GrpcStore::get_part()")), + } + } + let resource_name = if self.use_legacy_resource_names { format!( "{}/blobs/{}/{}", @@ -1013,7 +1374,7 @@ impl StoreDriver for GrpcStore { loop { let data = match stream.next().await { // Create an empty response to represent EOF. - None => bytes::Bytes::new(), + None => Bytes::new(), Some(Ok(message)) => message.data, Some(Err(status)) => { return Some(( diff --git a/nativelink-store/tests/grpc_read_batching_test.rs b/nativelink-store/tests/grpc_read_batching_test.rs new file mode 100644 index 000000000..86d419c3b --- /dev/null +++ b/nativelink-store/tests/grpc_read_batching_test.rs @@ -0,0 +1,572 @@ +// Copyright 2026 The NativeLink Authors. All rights reserved. +// +// Licensed under the Functional Source License, Version 1.1, Apache 2.0 Future License (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// See LICENSE file for details +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use core::future::Future; +use core::pin::Pin; +use core::task::{Context as TaskContext, Waker}; +use std::collections::{HashMap, HashSet}; +use std::sync::Arc; + +use async_lock::Mutex; +use bytes::Bytes; +use futures::Stream; +use nativelink_config::stores::{GrpcEndpoint, GrpcReadBatchingConfig, GrpcSpec, Retry, StoreType}; +use nativelink_error::{Code, Error}; +use nativelink_macro::nativelink_test; +use nativelink_proto::build::bazel::remote::execution::v2::content_addressable_storage_server::{ + ContentAddressableStorage, ContentAddressableStorageServer, +}; +use nativelink_proto::build::bazel::remote::execution::v2::{ + BatchReadBlobsRequest, BatchReadBlobsResponse, BatchUpdateBlobsRequest, + BatchUpdateBlobsResponse, FindMissingBlobsRequest, FindMissingBlobsResponse, GetTreeRequest, + GetTreeResponse, SpliceBlobRequest, SpliceBlobResponse, SplitBlobRequest, SplitBlobResponse, + batch_read_blobs_response, +}; +use nativelink_proto::google::bytestream::byte_stream_server::{ByteStream, ByteStreamServer}; +use nativelink_proto::google::bytestream::{ + QueryWriteStatusRequest, QueryWriteStatusResponse, ReadRequest, ReadResponse, WriteRequest, + WriteResponse, +}; +use nativelink_proto::google::rpc::Status as RpcStatus; +use nativelink_store::grpc_store::GrpcStore; +use nativelink_util::background_spawn; +use nativelink_util::common::DigestInfo; +use nativelink_util::store_trait::StoreLike; +use tonic::transport::Server; +use tonic::transport::server::TcpIncoming; +use tonic::{Request, Response, Status, Streaming}; + +/// A fake CAS server that serves seeded blobs from `BatchReadBlobs` and +/// records every `BatchReadBlobs` request it receives. Digests listed in +/// `not_found_hashes` are reported as `NOT_FOUND` in the per-blob status. +#[derive(Debug, Clone)] +struct FakeCasServer { + blobs: Arc>>, + not_found_hashes: Arc>>, + batch_read_requests: Arc>>, +} + +impl FakeCasServer { + fn new() -> Self { + Self { + blobs: Arc::new(Mutex::new(HashMap::new())), + not_found_hashes: Arc::new(Mutex::new(vec![])), + batch_read_requests: Arc::new(Mutex::new(vec![])), + } + } +} + +type GetTreeStream = Pin> + Send + 'static>>; + +#[tonic::async_trait] +impl ContentAddressableStorage for FakeCasServer { + type GetTreeStream = GetTreeStream; + + #[allow(clippy::unimplemented)] + async fn find_missing_blobs( + &self, + _grpc_request: Request, + ) -> Result, Status> { + unimplemented!(); + } + + #[allow(clippy::unimplemented)] + async fn batch_update_blobs( + &self, + _grpc_request: Request, + ) -> Result, Status> { + unimplemented!(); + } + + async fn batch_read_blobs( + &self, + grpc_request: Request, + ) -> Result, Status> { + let request = grpc_request.into_inner(); + self.batch_read_requests.lock().await.push(request.clone()); + + let blobs = self.blobs.lock().await; + let not_found_hashes = self.not_found_hashes.lock().await; + let mut responses = Vec::with_capacity(request.digests.len()); + // Real servers may dedupe duplicate digests in one request; mimic + // that by returning exactly one entry per unique digest. + let mut seen_hashes = HashSet::new(); + for digest in request.digests { + if !seen_hashes.insert(digest.hash.clone()) { + continue; + } + let (data, status) = if not_found_hashes.contains(&digest.hash) { + ( + Bytes::new(), + RpcStatus { + code: Code::NotFound as i32, + message: format!("Blob {} not found", digest.hash), + details: vec![], + }, + ) + } else if let Some(data) = blobs.get(&digest.hash) { + ( + data.clone(), + RpcStatus { + code: Code::Ok as i32, + message: String::new(), + details: vec![], + }, + ) + } else { + return Err(Status::invalid_argument(format!( + "Blob {} was never seeded in FakeCasServer", + digest.hash + ))); + }; + responses.push(batch_read_blobs_response::Response { + digest: Some(digest), + data, + compressor: 0, + status: Some(status), + }); + } + Ok(Response::new(BatchReadBlobsResponse { responses })) + } + + #[allow(clippy::unimplemented)] + async fn get_tree( + &self, + _grpc_request: Request, + ) -> Result, Status> { + unimplemented!(); + } + + #[allow(clippy::unimplemented)] + async fn split_blob( + &self, + _grpc_request: Request, + ) -> Result, Status> { + unimplemented!(); + } + + #[allow(clippy::unimplemented)] + async fn splice_blob( + &self, + _grpc_request: Request, + ) -> Result, Status> { + unimplemented!(); + } +} + +type ReadStream = Pin> + Send + 'static>>; + +/// A fake `ByteStream` server that serves the same seeded blobs as +/// [`FakeCasServer`] and records every `Read` request it receives. +#[derive(Debug, Clone)] +struct FakeByteStreamServer { + blobs: Arc>>, + read_requests: Arc>>, +} + +impl FakeByteStreamServer { + fn new(blobs: Arc>>) -> Self { + Self { + blobs, + read_requests: Arc::new(Mutex::new(vec![])), + } + } +} + +#[tonic::async_trait] +impl ByteStream for FakeByteStreamServer { + type ReadStream = ReadStream; + + async fn read( + &self, + grpc_request: Request, + ) -> Result, Status> { + let request = grpc_request.into_inner(); + self.read_requests.lock().await.push(request.clone()); + + // Resource names look like `{instance}/blobs/{fn}/{hash}/{size}`. + let mut components = request.resource_name.rsplit('/'); + let _size = components.next(); + let hash = components.next().unwrap_or_default().to_string(); + let data = self + .blobs + .lock() + .await + .get(&hash) + .cloned() + .ok_or_else(|| Status::not_found(format!("Blob {hash} not seeded")))?; + let offset = usize::try_from(request.read_offset) + .map_err(|_| Status::invalid_argument("Bad read_offset"))?; + if offset > data.len() { + return Err(Status::out_of_range("read_offset past end of blob")); + } + let stream: Self::ReadStream = Box::pin(futures::stream::iter(vec![Ok(ReadResponse { + data: data.slice(offset..), + })])); + Ok(Response::new(stream)) + } + + #[allow(clippy::unimplemented)] + async fn write( + &self, + _grpc_request: Request>, + ) -> Result, Status> { + unimplemented!(); + } + + #[allow(clippy::unimplemented)] + async fn query_write_status( + &self, + _grpc_request: Request, + ) -> Result, Status> { + unimplemented!(); + } +} + +struct TestFixture { + cas_server: FakeCasServer, + bytestream_server: FakeByteStreamServer, + store: Arc, +} + +async fn make_fixture(read_batching: Option) -> Result { + let cas_server = FakeCasServer::new(); + let bytestream_server = FakeByteStreamServer::new(cas_server.blobs.clone()); + let listener = TcpIncoming::bind("127.0.0.1:0".parse().unwrap()).unwrap(); + let port = listener.local_addr().unwrap().port(); + + let cas_service = ContentAddressableStorageServer::new(cas_server.clone()); + let bytestream_service = ByteStreamServer::new(bytestream_server.clone()); + background_spawn!("grpc_read_batching_test_server", async move { + Server::builder() + .add_service(cas_service) + .add_service(bytestream_service) + .serve_with_incoming(listener) + .await + .unwrap(); + }); + + let spec = GrpcSpec { + instance_name: String::new(), + endpoints: vec![GrpcEndpoint { + address: format!("http://localhost:{port}"), + tls_config: None, + concurrency_limit: None, + connect_timeout_s: 0, + tcp_keepalive_s: 0, + http2_keepalive_interval_s: 0, + http2_keepalive_timeout_s: 0, + }], + store_type: StoreType::Cas, + retry: Retry::default(), + max_concurrent_requests: 0, + connections_per_endpoint: 0, + rpc_timeout_s: 0, + use_legacy_resource_names: false, + headers: HashMap::new(), + forward_headers: vec![], + experimental_read_batching: read_batching, + }; + let store = GrpcStore::new(&spec).await?; + Ok(TestFixture { + cas_server, + bytestream_server, + store, + }) +} + +const fn batching_config() -> GrpcReadBatchingConfig { + GrpcReadBatchingConfig { + max_blob_size_bytes: 128 * 1024, + max_batch_bytes: 3 * 1024 * 1024, + dispatch_slots: 1, + max_queued_bytes: 32 * 1024 * 1024, + } +} + +/// Creates a unique digest and 64-byte content for blob index `i`. +fn make_blob(i: usize) -> (DigestInfo, Bytes) { + let hash = format!("{:064x}", i + 1); + let content = Bytes::from(format!("{i:0>64}")); + let digest = DigestInfo::try_new(&hash, content.len()).unwrap(); + (digest, content) +} + +async fn seed_blobs(fixture: &TestFixture, blobs: &[(DigestInfo, Bytes)]) { + let mut map = fixture.cas_server.blobs.lock().await; + for (digest, content) in blobs { + map.insert(digest.packed_hash().to_string(), content.clone()); + } +} + +// Many concurrent small reads must be coalesced into BatchReadBlobs RPCs +// and never touch the ByteStream Read path. +#[nativelink_test] +async fn small_blob_reads_are_batched() -> Result<(), Error> { + const NUM_BLOBS: usize = 200; + + let fixture = make_fixture(Some(batching_config())).await?; + let mut blobs = Vec::with_capacity(NUM_BLOBS); + for i in 0..NUM_BLOBS { + blobs.push(make_blob(i)); + } + seed_blobs(&fixture, &blobs).await; + + let mut read_futures = Vec::with_capacity(NUM_BLOBS); + for (digest, _) in &blobs { + read_futures.push(fixture.store.get_part_unchunked(*digest, 0, None)); + } + let results = futures::future::join_all(read_futures).await; + for (result, (_, expected_content)) in results.into_iter().zip(&blobs) { + assert_eq!(&result?, expected_content); + } + + let batch_read_requests = fixture.cas_server.batch_read_requests.lock().await; + assert!( + !batch_read_requests.is_empty(), + "Expected BatchReadBlobs to be used" + ); + let total_batched_digests: usize = batch_read_requests + .iter() + .map(|request| request.digests.len()) + .sum(); + assert_eq!(total_batched_digests, NUM_BLOBS); + assert!( + batch_read_requests.len() < NUM_BLOBS / 2, + "Expected reads to be coalesced, got {} BatchReadBlobs calls", + batch_read_requests.len() + ); + assert_eq!( + fixture.bytestream_server.read_requests.lock().await.len(), + 0, + "ByteStream Read must not be used for batched reads" + ); + Ok(()) +} + +// Blobs above max_blob_size_bytes must use the ByteStream Read path. +#[nativelink_test] +async fn large_blob_uses_stream_path() -> Result<(), Error> { + let mut config = batching_config(); + config.max_blob_size_bytes = 1024; + let fixture = make_fixture(Some(config)).await?; + + let hash = format!("{:064x}", 42_u32); + let content = Bytes::from(vec![42_u8; 2048]); + let digest = DigestInfo::try_new(&hash, content.len()).unwrap(); + seed_blobs(&fixture, &[(digest, content.clone())]).await; + + let data = fixture.store.get_part_unchunked(digest, 0, None).await?; + assert_eq!(data, content); + + assert_eq!( + fixture.bytestream_server.read_requests.lock().await.len(), + 1 + ); + assert_eq!(fixture.cas_server.batch_read_requests.lock().await.len(), 0); + Ok(()) +} + +// Partial reads (offset != 0) must bypass batching. +#[nativelink_test] +async fn partial_read_bypasses_batching() -> Result<(), Error> { + let fixture = make_fixture(Some(batching_config())).await?; + let (digest, content) = make_blob(7); + seed_blobs(&fixture, &[(digest, content.clone())]).await; + + let data = fixture.store.get_part_unchunked(digest, 1, None).await?; + assert_eq!(data, content.slice(1..)); + + let read_requests = fixture.bytestream_server.read_requests.lock().await; + assert_eq!(read_requests.len(), 1); + assert_eq!(read_requests[0].read_offset, 1); + assert_eq!(fixture.cas_server.batch_read_requests.lock().await.len(), 0); + Ok(()) +} + +// A NOT_FOUND for one digest in a batch must fail only that read; the +// other reads in the same batch must succeed. +#[nativelink_test] +async fn batch_item_failure_is_isolated() -> Result<(), Error> { + let fixture = make_fixture(Some(batching_config())).await?; + let (good_digest1, good_content1) = make_blob(1); + let (bad_digest, _) = make_blob(2); + let (good_digest2, good_content2) = make_blob(3); + seed_blobs( + &fixture, + &[ + (good_digest1, good_content1.clone()), + (good_digest2, good_content2.clone()), + ], + ) + .await; + fixture + .cas_server + .not_found_hashes + .lock() + .await + .push(bad_digest.packed_hash().to_string()); + + // On the single-threaded test runtime all three reads enqueue before + // the detached dispatcher task gets to run, so they coalesce. + let (good_result1, bad_result, good_result2) = futures::join!( + fixture.store.get_part_unchunked(good_digest1, 0, None), + fixture.store.get_part_unchunked(bad_digest, 0, None), + fixture.store.get_part_unchunked(good_digest2, 0, None), + ); + assert_eq!(good_result1?, good_content1); + assert_eq!(good_result2?, good_content2); + let bad_err = bad_result.expect_err("Expected NOT_FOUND digest to fail"); + assert_eq!(bad_err.code, Code::NotFound, "Got error: {bad_err:?}"); + + // Confirm that the failed digest actually shared a batch with another + // read (rather than each read getting its own RPC). + let batch_read_requests = fixture.cas_server.batch_read_requests.lock().await; + let bad_hash = bad_digest.packed_hash().to_string(); + assert!( + batch_read_requests + .iter() + .any(|request| request.digests.len() > 1 + && request.digests.iter().any(|d| d.hash == bad_hash)), + "Expected the failing digest to share a batch with other reads: {batch_read_requests:?}" + ); + assert_eq!( + fixture.bytestream_server.read_requests.lock().await.len(), + 0 + ); + Ok(()) +} + +// With batching unconfigured the store must behave exactly as before and +// never call BatchReadBlobs. +#[nativelink_test] +async fn disabled_batching_uses_stream_path() -> Result<(), Error> { + let fixture = make_fixture(None).await?; + let (digest, content) = make_blob(11); + seed_blobs(&fixture, &[(digest, content.clone())]).await; + + let data = fixture.store.get_part_unchunked(digest, 0, None).await?; + assert_eq!(data, content); + + assert_eq!( + fixture.bytestream_server.read_requests.lock().await.len(), + 1 + ); + assert_eq!(fixture.cas_server.batch_read_requests.lock().await.len(), 0); + Ok(()) +} + +// Cancelling one read mid-flight must not abort the shared batch RPC nor +// strand the other queued reads. +#[nativelink_test] +async fn cancelled_read_does_not_affect_batch_mates() -> Result<(), Error> { + let fixture = make_fixture(Some(batching_config())).await?; + let (cancelled_digest, cancelled_content) = make_blob(1); + let (digest_b, content_b) = make_blob(2); + let (digest_c, content_c) = make_blob(3); + seed_blobs( + &fixture, + &[ + (cancelled_digest, cancelled_content), + (digest_b, content_b.clone()), + (digest_c, content_c.clone()), + ], + ) + .await; + + // Poll one read just far enough to enqueue it, then drop it to + // simulate cancellation before its batch has been dispatched. + let waker = Waker::noop(); + let mut task_context = TaskContext::from_waker(waker); + let mut cancelled_read = Box::pin(fixture.store.get_part_unchunked(cancelled_digest, 0, None)); + assert!(cancelled_read.as_mut().poll(&mut task_context).is_pending()); + drop(cancelled_read); + + let (result_b, result_c) = futures::join!( + fixture.store.get_part_unchunked(digest_b, 0, None), + fixture.store.get_part_unchunked(digest_c, 0, None), + ); + assert_eq!(result_b?, content_b); + assert_eq!(result_c?, content_c); + assert_eq!( + fixture.bytestream_server.read_requests.lock().await.len(), + 0 + ); + Ok(()) +} + +// Concurrent reads of the same digest must all succeed even though the +// digest is requested only once per batch and the server responds with a +// single entry for it. +#[nativelink_test] +async fn duplicate_digest_reads_all_succeed() -> Result<(), Error> { + let fixture = make_fixture(Some(batching_config())).await?; + let (digest, content) = make_blob(5); + seed_blobs(&fixture, &[(digest, content.clone())]).await; + + let (result_a, result_b) = futures::join!( + fixture.store.get_part_unchunked(digest, 0, None), + fixture.store.get_part_unchunked(digest, 0, None), + ); + assert_eq!(result_a?, content); + assert_eq!(result_b?, content); + + // The client must dedupe: each request may carry the digest only once. + let batch_read_requests = fixture.cas_server.batch_read_requests.lock().await; + assert!(!batch_read_requests.is_empty()); + for request in batch_read_requests.iter() { + assert_eq!( + request.digests.len(), + 1, + "Expected deduped digests: {request:?}" + ); + } + assert_eq!( + fixture.bytestream_server.read_requests.lock().await.len(), + 0 + ); + Ok(()) +} + +// Batched reads share one upstream RPC, so combining them with per-client +// forwarded headers must be rejected at construction. +#[nativelink_test] +async fn forward_headers_with_batching_rejected() -> Result<(), Error> { + let spec = GrpcSpec { + instance_name: String::new(), + endpoints: vec![GrpcEndpoint { + address: "http://foobar".to_string(), + tls_config: None, + concurrency_limit: None, + connect_timeout_s: 0, + tcp_keepalive_s: 0, + http2_keepalive_interval_s: 0, + http2_keepalive_timeout_s: 0, + }], + store_type: StoreType::Cas, + retry: Retry::default(), + max_concurrent_requests: 0, + connections_per_endpoint: 0, + rpc_timeout_s: 0, + use_legacy_resource_names: false, + headers: HashMap::new(), + forward_headers: vec!["authorization".to_string()], + experimental_read_batching: Some(batching_config()), + }; + let err = GrpcStore::new(&spec) + .await + .expect_err("Expected construction to fail"); + assert_eq!(err.code, Code::InvalidArgument, "Got error: {err:?}"); + Ok(()) +} diff --git a/nativelink-store/tests/grpc_store_test.rs b/nativelink-store/tests/grpc_store_test.rs index 858d5a653..e120402d4 100644 --- a/nativelink-store/tests/grpc_store_test.rs +++ b/nativelink-store/tests/grpc_store_test.rs @@ -61,6 +61,7 @@ fn test_spec>(endpoint: T, use_legacy_resource_names: bool) -> G use_legacy_resource_names, headers: HashMap::new(), forward_headers: vec![], + experimental_read_batching: None, } } From e952657f92b360cb29dce51b5e220e0bf3a21b66 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Thu, 16 Jul 2026 15:38:37 +0100 Subject: [PATCH 06/84] docs(config-reference): regenerate for NativeLink (#2558) Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> --- .../docs/reference/nativelink-config/main.mdx | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/web/apps/docs/content/docs/reference/nativelink-config/main.mdx b/web/apps/docs/content/docs/reference/nativelink-config/main.mdx index adfbe5b45..39d0baf04 100644 --- a/web/apps/docs/content/docs/reference/nativelink-config/main.mdx +++ b/web/apps/docs/content/docs/reference/nativelink-config/main.mdx @@ -5,7 +5,7 @@ full: true --- {/* AUTOGENERATED — do not edit by hand. - Source: nativelink-config @ main (f871377c) + Source: nativelink-config @ main (4be35e6d) Regenerate from web/: bun --filter @nativelink/docs gen:config-reference */} @@ -852,6 +852,7 @@ See [`experimental_cloud_object_store`](#experimental_cloud_object_store-1) for | `use_legacy_resource_names` | boolean | — | false | Use legacy `ByteStream` resource name format, omitting the digest function component from the path. | | `headers` | map of string to string | — | — | Static headers to attach to every outgoing gRPC request sent to this store's upstream endpoints. Useful for fixed authentication tokens (e.g. `{"authorization": "Bearer "}`) and other static metadata. | | `forward_headers` | array of string | — | — | Header names to forward from the incoming client request to every outgoing upstream request. The header value is taken from the client request that triggered this store operation. Use this to pass through dynamic credentials such as JWT tokens sent by build clients. | +| `experimental_read_batching` | [GrpcReadBatchingConfig](#grpcreadbatchingconfig) | — | unset (disabled) | Optional and experimental: coalesce small-blob reads into `BatchReadBlobs` RPCs instead of issuing one `ByteStream` `Read` stream per blob. Each `ByteStream` read carries a fixed per-RPC cost, so batching many small reads into a single `BatchReadBlobs` request can dramatically reduce read latency for small blobs. | ## RedisSpec @@ -1137,6 +1138,18 @@ Configuration for an individual shard of the store. | `"cas"` | The store is content addressable storage. | | `"ac"` | The store is an action cache. | +## GrpcReadBatchingConfig + +Configuration for experimental small-blob read coalescing in a gRPC +store. See [`GrpcSpec::experimental_read_batching`]. + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `max_blob_size_bytes` | integer (uint64) | — | 131072 (128 KiB) | Only blobs at or below this size (in bytes) are eligible for batching. Larger blobs always use the `ByteStream` `Read` path. | +| `max_batch_bytes` | integer (uint64) | — | 3145728 (3 MiB) | Maximum total payload bytes packed into a single `BatchReadBlobs` request. This should leave headroom under the 4 MiB default gRPC message limit for protobuf framing overhead. | +| `dispatch_slots` | integer (uint) | — | 4 | Maximum number of concurrent `BatchReadBlobs` RPCs dispatched by the coalescer. Must be greater than zero. | +| `max_queued_bytes` | integer (uint64) | — | 33554432 (32 MiB) | Bound on the number of payload bytes waiting in the coalescer queue. When exceeded, new read requests bypass batching and fall back to the regular `ByteStream` `Read` path instead of blocking. | + ## RedisMode | Value | Description | From 7a4279b4e51cfb6064937b50f763e3c13733db0a Mon Sep 17 00:00:00 2001 From: Marcus Eagan Date: Thu, 16 Jul 2026 16:14:21 +0100 Subject: [PATCH 07/84] Add fallback match interval to simple scheduler (#2557) --- nativelink-config/src/schedulers.rs | 19 ++ nativelink-scheduler/Cargo.toml | 3 + nativelink-scheduler/src/simple_scheduler.rs | 24 +- .../tests/simple_scheduler_test.rs | 222 ++++++++++++++++++ 4 files changed, 265 insertions(+), 3 deletions(-) diff --git a/nativelink-config/src/schedulers.rs b/nativelink-config/src/schedulers.rs index 1f74832cd..3b840e545 100644 --- a/nativelink-config/src/schedulers.rs +++ b/nativelink-config/src/schedulers.rs @@ -83,6 +83,11 @@ const fn default_worker_match_logging_interval_s() -> i64 { 10 } +// defaults to every 5s +const fn default_fallback_match_interval_s() -> i64 { + 5 +} + #[derive(Deserialize, Serialize, Debug, Default)] #[serde(deny_unknown_fields)] #[cfg_attr(feature = "dev-schema", derive(JsonSchema))] @@ -167,6 +172,20 @@ pub struct SimpleSpec { deserialize_with = "convert_duration_with_shellexpand_and_negative" )] pub worker_match_logging_interval_s: i64, + + /// Every N seconds, run a worker matching pass even if no task or worker + /// change notification arrived. This is a safety net for missed + /// notifications and for scheduler backends with eventually consistent + /// searches (for example Redis), where an operation that was re-queued + /// may not be visible to the search triggered by its own notification. + /// Without this, such an operation can stay queued until an unrelated + /// event triggers another matching pass. + /// Defaults to 5s. Zero or any negative value disables it. + #[serde( + default = "default_fallback_match_interval_s", + deserialize_with = "convert_duration_with_shellexpand_and_negative" + )] + pub fallback_match_interval_s: i64, } #[derive(Deserialize, Serialize, Debug)] diff --git a/nativelink-scheduler/Cargo.toml b/nativelink-scheduler/Cargo.toml index 204c18e6a..43d97dad5 100644 --- a/nativelink-scheduler/Cargo.toml +++ b/nativelink-scheduler/Cargo.toml @@ -59,6 +59,9 @@ nativelink-redis-tester = { path = "../nativelink-redis-tester" } pretty_assertions = { version = "1.4.1", features = [ "std", ], default-features = false } +tokio = { version = "1.52.2", features = [ + "test-util", +], default-features = false } tracing-test = { version = "0.2.5", default-features = false, features = [ "no-env-filter", ] } diff --git a/nativelink-scheduler/src/simple_scheduler.rs b/nativelink-scheduler/src/simple_scheduler.rs index 0abd6bad7..97da8fe1c 100644 --- a/nativelink-scheduler/src/simple_scheduler.rs +++ b/nativelink-scheduler/src/simple_scheduler.rs @@ -532,6 +532,12 @@ impl SimpleScheduler { let worker_scheduler_clone = worker_scheduler.clone(); + let fallback_match_interval = match spec.fallback_match_interval_s { + // Zero or any negative value means disabled. + ..=0 => None, + secs => Some(Duration::from_secs(secs.unsigned_abs())), + }; + let action_scheduler = Arc::new_cyclic(move |weak_self| -> Self { let weak_inner = weak_self.clone(); let task_worker_matching_spawn = @@ -546,16 +552,28 @@ impl SimpleScheduler { tokio::pin!(worker_change_fut); // Wait for either of these futures to be ready. let state_changed = future::select(task_change_fut, worker_change_fut); - if last_match_successful { - let _ = state_changed.await; + let max_wait = if last_match_successful { + // Even on success, periodically re-run the match as a + // fallback for missed notifications and eventually + // consistent backends (e.g. a re-queued operation that + // was not yet visible to the search triggered by its + // own notification). Without this, such an operation + // can stay queued until an unrelated event triggers + // another matching pass. + fallback_match_interval } else { // If the last match failed, then run again after a short sleep. // This resolves issues where we tried to re-schedule a job to // a disconnected worker. The sleep ensures we don't enter a // hard loop if there's something wrong inside do_try_match. - let sleep_fut = tokio::time::sleep(Duration::from_millis(100)); + Some(Duration::from_millis(100)) + }; + if let Some(max_wait) = max_wait { + let sleep_fut = tokio::time::sleep(max_wait); tokio::pin!(sleep_fut); let _ = future::select(state_changed, sleep_fut).await; + } else { + let _ = state_changed.await; } let result = match weak_inner.upgrade() { diff --git a/nativelink-scheduler/tests/simple_scheduler_test.rs b/nativelink-scheduler/tests/simple_scheduler_test.rs index 2938304d1..3b6bf7766 100644 --- a/nativelink-scheduler/tests/simple_scheduler_test.rs +++ b/nativelink-scheduler/tests/simple_scheduler_test.rs @@ -2789,3 +2789,225 @@ async fn logs_when_no_workers_match() -> Result<(), Error> { Ok(()) } + +/// Wraps a real `AwaitedActionDb`, but hides queued actions from +/// `get_range_of_actions` while `suppress_queued_searches` is set. This +/// simulates an eventually consistent backend (e.g. Redis), where a +/// (re-)queued operation may not yet be visible to the search that its own +/// change notification triggered. +#[derive(MetricsComponent)] +struct QueuedSearchSuppressingDb { + inner: A, + suppress_queued_searches: Arc, +} + +impl AwaitedActionDb for QueuedSearchSuppressingDb { + type Subscriber = A::Subscriber; + + async fn get_awaited_action_by_id( + &self, + client_operation_id: &OperationId, + ) -> Result, Error> { + self.inner + .get_awaited_action_by_id(client_operation_id) + .await + } + + async fn get_all_awaited_actions( + &self, + ) -> Result> + Send, Error> { + self.inner.get_all_awaited_actions().await + } + + async fn get_by_operation_id( + &self, + operation_id: &OperationId, + ) -> Result, Error> { + self.inner.get_by_operation_id(operation_id).await + } + + async fn get_range_of_actions( + &self, + state: SortedAwaitedActionState, + start: Bound, + end: Bound, + desc: bool, + ) -> Result> + Send, Error> { + let items = if matches!(state, SortedAwaitedActionState::Queued) + && self.suppress_queued_searches.load(Ordering::Acquire) + { + Vec::new() + } else { + self.inner + .get_range_of_actions(state, start, end, desc) + .await? + .collect::>() + .await + }; + Ok(futures::stream::iter(items)) + } + + async fn update_awaited_action(&self, new_awaited_action: AwaitedAction) -> Result<(), Error> { + self.inner.update_awaited_action(new_awaited_action).await + } + + async fn add_action( + &self, + client_operation_id: OperationId, + action_info: Arc, + no_event_action_timeout: Duration, + ) -> Result { + self.inner + .add_action(client_operation_id, action_info, no_event_action_timeout) + .await + } +} + +/// Common setup for the fallback match interval tests: a scheduler over a +/// `QueuedSearchSuppressingDb` with a channel that receives a message after +/// every completed matching pass. +type FallbackTestSetup = ( + Arc, + Arc, + Arc, + mpsc::UnboundedReceiver<()>, +); + +fn make_fallback_test_scheduler(fallback_match_interval_s: i64) -> FallbackTestSetup { + let task_change_notify = Arc::new(Notify::new()); + let suppress_queued_searches = Arc::new(AtomicBool::new(false)); + let (match_tx, match_rx) = mpsc::unbounded_channel(); + let (scheduler, _worker_scheduler) = SimpleScheduler::new_with_callback( + &SimpleSpec { + fallback_match_interval_s, + ..Default::default() + }, + QueuedSearchSuppressingDb { + inner: memory_awaited_action_db_factory( + 0, + &task_change_notify.clone(), + MockInstantWrapped::default, + ), + suppress_queued_searches: suppress_queued_searches.clone(), + }, + move || { + let match_tx = match_tx.clone(); + async move { + let _ = match_tx.send(()); + } + }, + task_change_notify.clone(), + MockInstantWrapped::default, + None, + ); + ( + scheduler, + task_change_notify, + suppress_queued_searches, + match_rx, + ) +} + +/// Waits until no matching pass has completed for a short while, which +/// guarantees no notification permits are pending and no pass is in flight. +async fn wait_for_matching_passes_to_settle(match_rx: &mut mpsc::UnboundedReceiver<()>) { + while tokio::time::timeout(Duration::from_millis(200), match_rx.recv()) + .await + .is_ok() + {} +} + +// Regression test for a queued operation not being visible to the matching +// engine search that its own change notification triggered (e.g. an OOM-killed +// worker's operation being re-queued while the Redis search index is stale). +// The fallback match interval must rescue such an operation. +#[nativelink_test(start_paused = true)] +async fn fallback_match_interval_rescues_action_hidden_from_search() -> Result<(), Error> { + let worker_id = WorkerId("worker_id".to_string()); + let (scheduler, _task_change_notify, suppress_queued_searches, mut match_rx) = + make_fallback_test_scheduler(1 /* fallback_match_interval_s */); + let action_digest = DigestInfo::new([99u8; 32], 512); + + let mut rx_from_worker = + setup_new_worker(&scheduler, worker_id.clone(), PlatformProperties::default()).await?; + + // Hide the action from the matching engine's queued searches, then add it. + suppress_queued_searches.store(true, Ordering::Release); + let insert_timestamp = make_system_time(1); + let mut action_listener = + setup_action(&scheduler, action_digest, HashMap::new(), insert_timestamp).await?; + + wait_for_matching_passes_to_settle(&mut match_rx).await; + + // All notification-triggered passes ran while the action was hidden, so + // nothing was assigned to the worker. + assert_eq!(poll!(Box::pin(rx_from_worker.recv())), Poll::Pending); + + // Make the action visible again. No notification fires for this, so only + // the fallback match interval can rescue the action now. + suppress_queued_searches.store(false, Ordering::Release); + + let msg_for_worker = tokio::time::timeout(Duration::from_secs(30), rx_from_worker.recv()) + .await + .expect("Fallback matching pass should have assigned the action") + .unwrap(); + match msg_for_worker.update { + Some(update_for_worker::Update::StartAction(start_execute)) => { + assert_eq!( + start_execute.execute_request.unwrap().action_digest, + Some(action_digest.into()) + ); + } + other => panic!("Expected StartAction, got: {other:?}"), + } + + // Client should see the action executing. + let (action_state, _maybe_origin_metadata) = action_listener.changed().await.unwrap(); + assert_eq!(action_state.stage, ActionStage::Executing); + + Ok(()) +} + +// With the fallback match interval disabled, the same scenario leaves the +// action stuck in the queued state until an unrelated event triggers another +// matching pass. This documents the behavior the fallback interval fixes. +#[nativelink_test(start_paused = true)] +async fn fallback_match_interval_disabled_leaves_hidden_action_queued() -> Result<(), Error> { + let worker_id = WorkerId("worker_id".to_string()); + let (scheduler, task_change_notify, suppress_queued_searches, mut match_rx) = + make_fallback_test_scheduler(-1 /* fallback_match_interval_s */); + let action_digest = DigestInfo::new([99u8; 32], 512); + + let mut rx_from_worker = + setup_new_worker(&scheduler, worker_id.clone(), PlatformProperties::default()).await?; + + // Hide the action from the matching engine's queued searches, then add it. + suppress_queued_searches.store(true, Ordering::Release); + let insert_timestamp = make_system_time(1); + let _action_listener = + setup_action(&scheduler, action_digest, HashMap::new(), insert_timestamp).await?; + + wait_for_matching_passes_to_settle(&mut match_rx).await; + suppress_queued_searches.store(false, Ordering::Release); + + // Without the fallback interval nothing ever rescues the action. + assert!( + tokio::time::timeout(Duration::from_secs(30), rx_from_worker.recv()) + .await + .is_err(), + "Action should stay queued with the fallback match interval disabled" + ); + + // Only an unrelated task change event triggers another matching pass. + task_change_notify.notify_one(); + let msg_for_worker = tokio::time::timeout(Duration::from_secs(5), rx_from_worker.recv()) + .await + .expect("Task change notification should have assigned the action") + .unwrap(); + assert!(matches!( + msg_for_worker.update, + Some(update_for_worker::Update::StartAction(_)) + )); + + Ok(()) +} From 50036764f6e87fe0518f62eb2ee6b2924f7fadd7 Mon Sep 17 00:00:00 2001 From: Tom Parker-Shemilt Date: Thu, 16 Jul 2026 19:31:28 +0100 Subject: [PATCH 08/84] Tag config regen builds to our bot account (#2554) --- .github/workflows/config-reference.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/config-reference.yaml b/.github/workflows/config-reference.yaml index 965209f43..74eca2d5b 100644 --- a/.github/workflows/config-reference.yaml +++ b/.github/workflows/config-reference.yaml @@ -61,7 +61,7 @@ jobs: else TAG="${{ github.ref_name }}" fi - if [[ "${TAG}" != "" && ( ! "${TAG}" =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ) ]]; then + if [[ "${TAG}" != "" && "${TAG}" != "main" && ( ! "${TAG}" =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ) ]]; then echo "::error::'${TAG}' is not a release tag (vMAJOR.MINOR.PATCH); refusing to regenerate." exit 1 fi @@ -109,8 +109,8 @@ jobs: fi BRANCH="docs/config-reference-${TAG}" - git config user.name "github-actions[bot]" - git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + git config user.name "config reference bot" + git config user.email "bot@tracemachina.com" git switch -c "${BRANCH}" git add web/apps/docs git commit -m "docs(config-reference): regenerate for NativeLink ${TAG}" From 48d9b961e5403a904708e3394eead2a26df30d81 Mon Sep 17 00:00:00 2001 From: Tom Parker-Shemilt Date: Thu, 16 Jul 2026 19:58:30 +0100 Subject: [PATCH 09/84] Don't update config reference without changes (#2562) --- .../docs/scripts/gen-config-reference.mjs | 22 +++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/web/apps/docs/scripts/gen-config-reference.mjs b/web/apps/docs/scripts/gen-config-reference.mjs index 051cc4859..610fb256a 100644 --- a/web/apps/docs/scripts/gen-config-reference.mjs +++ b/web/apps/docs/scripts/gen-config-reference.mjs @@ -26,7 +26,7 @@ // the worktrees don't each spend disk on a full target/ tree. import { execFileSync } from "node:child_process"; -import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync, existsSync } from "node:fs"; import { tmpdir } from "node:os"; import { dirname, join } from "node:path"; import { fileURLToPath } from "node:url"; @@ -193,6 +193,17 @@ function generateOne(v, sharedTargetDir) { sourceDir = worktree; } + const outfile = outFileFor(v); + let existingMdx = ""; + if (existsSync(outfile)) { + existingMdx = readFileSync(outfile, { encoding: 'utf8', flag: 'r' }); + } + const existingVersionMatches = existingMdx.match(/Source: nativelink-config @ (.+)/); + var existingVersion = ""; + if (existingVersionMatches !== null) { + existingVersion = existingVersionMatches[1]; + } + try { const schema = buildSchema(sourceDir, v.isDev ? null : sharedTargetDir); const mdx = schemaToMdx(schema, { @@ -203,7 +214,14 @@ function generateOne(v, sharedTargetDir) { switcher: "", githubBase: GITHUB_BASE, }); - writeFileSync(outFileFor(v), mdx); + const newVersion = mdx.match(/Source: nativelink-config @ (.+)/)[1] + + if (mdx.replace(newVersion, existingVersion) != existingMdx) { + console.log(`Diff between '${existingVersion}' and '${newVersion}', so writing file`); + writeFileSync(outfile, mdx); + } else { + console.log(`No diff between '${existingVersion}' and '${newVersion}', so not writing file`); + } return { ...v, commit, defs: Object.keys(schema.$defs ?? schema.definitions ?? {}).length }; } finally { if (worktree) { From b18311f46f341bf30e4067a04b9ed93866b9bd46 Mon Sep 17 00:00:00 2001 From: Tom Parker-Shemilt Date: Thu, 16 Jul 2026 20:47:53 +0100 Subject: [PATCH 10/84] Upgrade serde_with to 3.21 for GHSA-7gcf-g7xr-8hxj (#2559) --- Cargo.lock | 30 +++++++++++++++--------------- MODULE.bazel.lock | 14 +++++++------- 2 files changed, 22 insertions(+), 22 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index dc92c0c58..2b405f954 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1328,9 +1328,9 @@ dependencies = [ [[package]] name = "darling" -version = "0.21.3" +version = "0.23.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9cdf337090841a411e2a7f3deb9187445851f91b309c0c0a29e05f74a00a48c0" +checksum = "25ae13da2f202d56bd7f91c25fba009e7717a1e4a1cc98a76d844b65ae912e9d" dependencies = [ "darling_core", "darling_macro", @@ -1338,11 +1338,10 @@ dependencies = [ [[package]] name = "darling_core" -version = "0.21.3" +version = "0.23.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1247195ecd7e3c85f83c8d2a366e4210d588e802133e1e355180a9870b517ea4" +checksum = "9865a50f7c335f53564bb694ef660825eb8610e0a53d3e11bf1b0d3df31e03b0" dependencies = [ - "fnv", "ident_case", "proc-macro2", "quote", @@ -1352,9 +1351,9 @@ dependencies = [ [[package]] name = "darling_macro" -version = "0.21.3" +version = "0.23.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d38308df82d1080de0afee5d069fa14b0326a88c14f15c5ccda35b4a6c414c81" +checksum = "ac3984ec7bd6cfa798e62b4a642426a5be0e68f9401cfc2a01e3fa9ea2fcdb8d" dependencies = [ "darling_core", "quote", @@ -4645,11 +4644,12 @@ dependencies = [ [[package]] name = "serde_with" -version = "3.15.1" +version = "3.21.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aa66c845eee442168b2c8134fec70ac50dc20e760769c8ba0ad1319ca1959b04" +checksum = "76a5c54c7310e7b8b9577c286d7e399ddd876c3e12b3ed917a8aabc4b96e9e8c" dependencies = [ "base64 0.22.1", + "bs58", "chrono", "hex", "indexmap 1.9.3", @@ -4664,9 +4664,9 @@ dependencies = [ [[package]] name = "serde_with_macros" -version = "3.15.1" +version = "3.21.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b91a903660542fced4e99881aa481bdbaec1634568ee02e0b8bd57c64cb38955" +checksum = "84d57bc0c8b9a17920c178daa6bb924850d54a9c97ab45194bb8c17ad66bb660" dependencies = [ "darling", "proc-macro2", @@ -5837,9 +5837,9 @@ dependencies = [ [[package]] name = "wincode" -version = "0.5.4" +version = "0.5.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "37095eb18dd6254c66217edc61a29d83d51f8818de8a2ffe88e4584ad73fb5f9" +checksum = "66d967db7705dc29120bb6e8ce5b5a2e27734ed5976d1c904e95bd238d1c3c5a" dependencies = [ "pastey", "proc-macro2", @@ -5850,9 +5850,9 @@ dependencies = [ [[package]] name = "wincode-derive" -version = "0.4.5" +version = "0.4.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e262d55d1261f31e2cfe49cc6385a421d14d99faa0526bbe3cc1bda0d3005c62" +checksum = "15ab90b719560d0fda79c74550ad1c948d17b118765942838055ebaf34d67071" dependencies = [ "darling", "proc-macro2", diff --git a/MODULE.bazel.lock b/MODULE.bazel.lock index 9b7bf7678..c1eef96bc 100644 --- a/MODULE.bazel.lock +++ b/MODULE.bazel.lock @@ -900,9 +900,9 @@ "crypto-common_0.1.6": "{\"dependencies\":[{\"features\":[\"more_lengths\"],\"name\":\"generic-array\",\"req\":\"^0.14.4\"},{\"name\":\"rand_core\",\"optional\":true,\"req\":\"^0.6\"},{\"name\":\"typenum\",\"req\":\"^1.14\"}],\"features\":{\"getrandom\":[\"rand_core/getrandom\"],\"std\":[]}}", "curve25519-dalek-derive_0.1.1": "{\"dependencies\":[{\"name\":\"proc-macro2\",\"req\":\"^1.0.66\"},{\"name\":\"quote\",\"req\":\"^1.0.31\"},{\"features\":[\"full\"],\"name\":\"syn\",\"req\":\"^2.0.27\"}],\"features\":{}}", "curve25519-dalek_4.1.3": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"bincode\",\"req\":\"^1\"},{\"name\":\"cfg-if\",\"req\":\"^1\"},{\"name\":\"cpufeatures\",\"req\":\"^0.2.6\",\"target\":\"cfg(target_arch = \\\"x86_64\\\")\"},{\"features\":[\"html_reports\"],\"kind\":\"dev\",\"name\":\"criterion\",\"req\":\"^0.5\"},{\"name\":\"curve25519-dalek-derive\",\"req\":\"^0.1\",\"target\":\"cfg(all(not(curve25519_dalek_backend = \\\"fiat\\\"), not(curve25519_dalek_backend = \\\"serial\\\"), target_arch = \\\"x86_64\\\"))\"},{\"default_features\":false,\"name\":\"digest\",\"optional\":true,\"req\":\"^0.10\"},{\"default_features\":false,\"name\":\"ff\",\"optional\":true,\"req\":\"^0.13\"},{\"default_features\":false,\"name\":\"fiat-crypto\",\"req\":\"^0.2.1\",\"target\":\"cfg(curve25519_dalek_backend = \\\"fiat\\\")\"},{\"default_features\":false,\"name\":\"group\",\"optional\":true,\"req\":\"^0.13\"},{\"kind\":\"dev\",\"name\":\"hex\",\"req\":\"^0.4.2\"},{\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.8\"},{\"default_features\":false,\"name\":\"rand_core\",\"optional\":true,\"req\":\"^0.6.4\"},{\"default_features\":false,\"features\":[\"getrandom\"],\"kind\":\"dev\",\"name\":\"rand_core\",\"req\":\"^0.6\"},{\"kind\":\"build\",\"name\":\"rustc_version\",\"req\":\"^0.4.0\"},{\"default_features\":false,\"features\":[\"derive\"],\"name\":\"serde\",\"optional\":true,\"req\":\"^1.0\"},{\"default_features\":false,\"kind\":\"dev\",\"name\":\"sha2\",\"req\":\"^0.10\"},{\"default_features\":false,\"name\":\"subtle\",\"req\":\"^2.3.0\"},{\"default_features\":false,\"name\":\"zeroize\",\"optional\":true,\"req\":\"^1\"}],\"features\":{\"alloc\":[\"zeroize?/alloc\"],\"default\":[\"alloc\",\"precomputed-tables\",\"zeroize\"],\"group\":[\"dep:group\",\"rand_core\"],\"group-bits\":[\"group\",\"ff/bits\"],\"legacy_compatibility\":[],\"precomputed-tables\":[]}}", - "darling_0.21.3": "{\"dependencies\":[{\"name\":\"darling_core\",\"req\":\"=0.21.3\"},{\"name\":\"darling_macro\",\"req\":\"=0.21.3\"},{\"kind\":\"dev\",\"name\":\"proc-macro2\",\"req\":\"^1.0.86\"},{\"kind\":\"dev\",\"name\":\"quote\",\"req\":\"^1.0.18\"},{\"kind\":\"dev\",\"name\":\"rustversion\",\"req\":\"^1.0.9\",\"target\":\"cfg(compiletests)\"},{\"kind\":\"dev\",\"name\":\"syn\",\"req\":\"^2.0.15\"},{\"kind\":\"dev\",\"name\":\"trybuild\",\"req\":\"^1.0.89\",\"target\":\"cfg(compiletests)\"}],\"features\":{\"default\":[\"suggestions\"],\"diagnostics\":[\"darling_core/diagnostics\"],\"serde\":[\"darling_core/serde\"],\"suggestions\":[\"darling_core/suggestions\"]}}", - "darling_core_0.21.3": "{\"dependencies\":[{\"name\":\"fnv\",\"req\":\"^1.0.7\"},{\"name\":\"ident_case\",\"req\":\"^1.0.1\"},{\"name\":\"proc-macro2\",\"req\":\"^1.0.86\"},{\"name\":\"quote\",\"req\":\"^1.0.18\"},{\"name\":\"serde\",\"optional\":true,\"req\":\"^1.0.210\"},{\"kind\":\"dev\",\"name\":\"serde_json\",\"req\":\"^1.0.140\"},{\"name\":\"strsim\",\"optional\":true,\"req\":\"^0.11.1\"},{\"features\":[\"full\",\"extra-traits\"],\"name\":\"syn\",\"req\":\"^2.0.15\"}],\"features\":{\"diagnostics\":[],\"suggestions\":[\"strsim\"]}}", - "darling_macro_0.21.3": "{\"dependencies\":[{\"name\":\"darling_core\",\"req\":\"=0.21.3\"},{\"name\":\"quote\",\"req\":\"^1.0.18\"},{\"name\":\"syn\",\"req\":\"^2.0.15\"}],\"features\":{}}", + "darling_0.23.0": "{\"dependencies\":[{\"name\":\"darling_core\",\"req\":\"=0.23.0\"},{\"name\":\"darling_macro\",\"req\":\"=0.23.0\"},{\"kind\":\"dev\",\"name\":\"proc-macro2\",\"req\":\"^1.0.86\"},{\"kind\":\"dev\",\"name\":\"quote\",\"req\":\"^1.0.18\"},{\"kind\":\"dev\",\"name\":\"rustversion\",\"req\":\"^1.0.9\",\"target\":\"cfg(compiletests)\"},{\"kind\":\"dev\",\"name\":\"syn\",\"req\":\"^2.0.15\"},{\"kind\":\"dev\",\"name\":\"trybuild\",\"req\":\"^1.0.89\",\"target\":\"cfg(compiletests)\"}],\"features\":{\"default\":[\"suggestions\"],\"diagnostics\":[\"darling_core/diagnostics\"],\"serde\":[\"darling_core/serde\"],\"suggestions\":[\"darling_core/suggestions\"]}}", + "darling_core_0.23.0": "{\"dependencies\":[{\"name\":\"ident_case\",\"req\":\"^1.0.1\"},{\"name\":\"proc-macro2\",\"req\":\"^1.0.86\"},{\"name\":\"quote\",\"req\":\"^1.0.18\"},{\"name\":\"serde\",\"optional\":true,\"req\":\"^1.0.210\"},{\"kind\":\"dev\",\"name\":\"serde_json\",\"req\":\"^1.0.140\"},{\"name\":\"strsim\",\"optional\":true,\"req\":\"^0.11.1\"},{\"features\":[\"full\",\"extra-traits\"],\"name\":\"syn\",\"req\":\"^2.0.15\"}],\"features\":{\"diagnostics\":[],\"suggestions\":[\"strsim\"]}}", + "darling_macro_0.23.0": "{\"dependencies\":[{\"name\":\"darling_core\",\"req\":\"=0.23.0\"},{\"name\":\"quote\",\"req\":\"^1.0.18\"},{\"name\":\"syn\",\"req\":\"^2.0.15\"}],\"features\":{}}", "data-encoding_2.11.0": "{\"dependencies\":[],\"features\":{\"alloc\":[],\"default\":[\"std\"],\"std\":[\"alloc\"]}}", "der_0.7.10": "{\"dependencies\":[{\"features\":[\"derive\"],\"name\":\"arbitrary\",\"optional\":true,\"req\":\"^1.3\"},{\"default_features\":false,\"name\":\"bytes\",\"optional\":true,\"req\":\"^1\"},{\"name\":\"const-oid\",\"optional\":true,\"req\":\"^0.9.2\"},{\"name\":\"der_derive\",\"optional\":true,\"req\":\"^0.7.2\"},{\"name\":\"flagset\",\"optional\":true,\"req\":\"^0.4.3\"},{\"kind\":\"dev\",\"name\":\"hex-literal\",\"req\":\"^0.4.1\"},{\"features\":[\"alloc\"],\"name\":\"pem-rfc7468\",\"optional\":true,\"req\":\"^0.7\"},{\"kind\":\"dev\",\"name\":\"proptest\",\"req\":\"^1\"},{\"default_features\":false,\"name\":\"time\",\"optional\":true,\"req\":\"^0.3.4\"},{\"default_features\":false,\"name\":\"zeroize\",\"optional\":true,\"req\":\"^1.5\"}],\"features\":{\"alloc\":[\"zeroize?/alloc\"],\"arbitrary\":[\"dep:arbitrary\",\"const-oid?/arbitrary\",\"std\"],\"bytes\":[\"dep:bytes\",\"alloc\"],\"derive\":[\"dep:der_derive\"],\"oid\":[\"dep:const-oid\"],\"pem\":[\"dep:pem-rfc7468\",\"alloc\",\"zeroize\"],\"real\":[],\"std\":[\"alloc\"]}}", "deranged_0.5.8": "{\"dependencies\":[{\"name\":\"deranged-macros\",\"optional\":true,\"req\":\"=0.3.0\"},{\"default_features\":false,\"name\":\"num-traits\",\"optional\":true,\"req\":\"^0.2.15\"},{\"default_features\":false,\"name\":\"powerfmt\",\"optional\":true,\"req\":\"^0.2.0\"},{\"default_features\":false,\"name\":\"quickcheck\",\"optional\":true,\"req\":\"^1.0.3\"},{\"default_features\":false,\"name\":\"rand010\",\"optional\":true,\"package\":\"rand\",\"req\":\"^0.10.0\"},{\"kind\":\"dev\",\"name\":\"rand010\",\"package\":\"rand\",\"req\":\"^0.10.0\"},{\"default_features\":false,\"name\":\"rand08\",\"optional\":true,\"package\":\"rand\",\"req\":\"^0.8.4\"},{\"kind\":\"dev\",\"name\":\"rand08\",\"package\":\"rand\",\"req\":\"^0.8.4\"},{\"default_features\":false,\"name\":\"rand09\",\"optional\":true,\"package\":\"rand\",\"req\":\"^0.9.0\"},{\"kind\":\"dev\",\"name\":\"rand09\",\"package\":\"rand\",\"req\":\"^0.9.0\"},{\"default_features\":false,\"name\":\"serde_core\",\"optional\":true,\"req\":\"^1.0.220\"},{\"kind\":\"dev\",\"name\":\"serde_json\",\"req\":\"^1.0.86\"}],\"features\":{\"alloc\":[],\"default\":[],\"macros\":[\"dep:deranged-macros\"],\"num\":[\"dep:num-traits\"],\"powerfmt\":[\"dep:powerfmt\"],\"quickcheck\":[\"dep:quickcheck\",\"alloc\"],\"rand\":[\"rand08\",\"rand09\",\"rand010\"],\"rand010\":[\"dep:rand010\"],\"rand08\":[\"dep:rand08\"],\"rand09\":[\"dep:rand09\"],\"serde\":[\"dep:serde_core\"]}}", @@ -1216,8 +1216,8 @@ "serde_json_1.0.150": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"automod\",\"req\":\"^1.0.11\"},{\"name\":\"indexmap\",\"optional\":true,\"req\":\"^2.2.3\"},{\"kind\":\"dev\",\"name\":\"indoc\",\"req\":\"^2.0.2\"},{\"name\":\"itoa\",\"req\":\"^1.0\"},{\"default_features\":false,\"name\":\"memchr\",\"req\":\"^2\"},{\"kind\":\"dev\",\"name\":\"ref-cast\",\"req\":\"^1.0.18\"},{\"kind\":\"dev\",\"name\":\"rustversion\",\"req\":\"^1.0.13\"},{\"default_features\":false,\"name\":\"serde\",\"req\":\"^1.0.220\",\"target\":\"cfg(any())\"},{\"features\":[\"derive\"],\"kind\":\"dev\",\"name\":\"serde\",\"req\":\"^1.0.194\"},{\"kind\":\"dev\",\"name\":\"serde_bytes\",\"req\":\"^0.11.10\"},{\"default_features\":false,\"name\":\"serde_core\",\"req\":\"^1.0.220\"},{\"kind\":\"dev\",\"name\":\"serde_derive\",\"req\":\"^1.0.166\"},{\"kind\":\"dev\",\"name\":\"serde_stacker\",\"req\":\"^0.1.8\"},{\"features\":[\"diff\"],\"kind\":\"dev\",\"name\":\"trybuild\",\"req\":\"^1.0.108\"},{\"name\":\"zmij\",\"req\":\"^1.0\"}],\"features\":{\"alloc\":[\"serde_core/alloc\"],\"arbitrary_precision\":[],\"default\":[\"std\"],\"float_roundtrip\":[],\"preserve_order\":[\"indexmap\",\"std\"],\"raw_value\":[],\"std\":[\"memchr/std\",\"serde_core/std\"],\"unbounded_depth\":[]}}", "serde_test_1.0.177": "{\"dependencies\":[{\"name\":\"serde\",\"req\":\"^1.0.69\"},{\"features\":[\"rc\"],\"kind\":\"dev\",\"name\":\"serde\",\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"serde_derive\",\"req\":\"^1\"}],\"features\":{}}", "serde_urlencoded_0.7.1": "{\"dependencies\":[{\"name\":\"form_urlencoded\",\"req\":\"^1\"},{\"name\":\"itoa\",\"req\":\"^1\"},{\"name\":\"ryu\",\"req\":\"^1\"},{\"name\":\"serde\",\"req\":\"^1.0.69\"},{\"kind\":\"dev\",\"name\":\"serde_derive\",\"req\":\"^1\"}],\"features\":{}}", - "serde_with_3.15.1": "{\"dependencies\":[{\"default_features\":false,\"name\":\"base64\",\"optional\":true,\"req\":\"^0.22.1\"},{\"default_features\":false,\"features\":[\"serde\"],\"name\":\"chrono_0_4\",\"optional\":true,\"package\":\"chrono\",\"req\":\"^0.4.20\"},{\"name\":\"document-features\",\"optional\":true,\"req\":\"^0.2.7\"},{\"kind\":\"dev\",\"name\":\"expect-test\",\"req\":\"^1.5.1\"},{\"kind\":\"dev\",\"name\":\"fnv\",\"req\":\"^1.0.6\"},{\"kind\":\"dev\",\"name\":\"glob\",\"req\":\"^0.3.3\"},{\"default_features\":false,\"features\":[\"serde\"],\"name\":\"hashbrown_0_14\",\"optional\":true,\"package\":\"hashbrown\",\"req\":\"^0.14.0\"},{\"default_features\":false,\"features\":[\"serde\"],\"name\":\"hashbrown_0_15\",\"optional\":true,\"package\":\"hashbrown\",\"req\":\"^0.15.0\"},{\"default_features\":false,\"features\":[\"serde\"],\"name\":\"hashbrown_0_16\",\"optional\":true,\"package\":\"hashbrown\",\"req\":\"^0.16.0\"},{\"default_features\":false,\"name\":\"hex\",\"optional\":true,\"req\":\"^0.4.3\"},{\"default_features\":false,\"features\":[\"serde-1\"],\"name\":\"indexmap_1\",\"optional\":true,\"package\":\"indexmap\",\"req\":\"^1.8\"},{\"default_features\":false,\"features\":[\"serde\"],\"name\":\"indexmap_2\",\"optional\":true,\"package\":\"indexmap\",\"req\":\"^2.0\"},{\"default_features\":false,\"features\":[\"resolve-file\"],\"kind\":\"dev\",\"name\":\"jsonschema\",\"req\":\"^0.33.0\"},{\"kind\":\"dev\",\"name\":\"mime\",\"req\":\"^0.3.16\"},{\"kind\":\"dev\",\"name\":\"pretty_assertions\",\"req\":\"^1.4.0\"},{\"default_features\":false,\"features\":[\"std\"],\"kind\":\"dev\",\"name\":\"regex\",\"req\":\"^1.12.1\"},{\"kind\":\"dev\",\"name\":\"rmp-serde\",\"req\":\"^1.3.0\"},{\"kind\":\"dev\",\"name\":\"ron\",\"req\":\"^0.11\"},{\"kind\":\"dev\",\"name\":\"rustversion\",\"req\":\"^1.0.22\"},{\"default_features\":false,\"name\":\"schemars_0_8\",\"optional\":true,\"package\":\"schemars\",\"req\":\"^0.8.16\"},{\"kind\":\"dev\",\"name\":\"schemars_0_8\",\"package\":\"schemars\",\"req\":\"^0.8.16\"},{\"default_features\":false,\"name\":\"schemars_0_9\",\"optional\":true,\"package\":\"schemars\",\"req\":\"^0.9.0\"},{\"kind\":\"dev\",\"name\":\"schemars_0_9\",\"package\":\"schemars\",\"req\":\"^0.9.0\"},{\"default_features\":false,\"name\":\"schemars_1\",\"optional\":true,\"package\":\"schemars\",\"req\":\"^1.0.2\"},{\"kind\":\"dev\",\"name\":\"schemars_1\",\"package\":\"schemars\",\"req\":\"^1.0.2\"},{\"default_features\":false,\"features\":[\"derive\"],\"kind\":\"dev\",\"name\":\"serde\",\"req\":\"^1.0.152\"},{\"kind\":\"dev\",\"name\":\"serde-xml-rs\",\"req\":\"^0.8.1\"},{\"default_features\":false,\"features\":[\"result\"],\"name\":\"serde_core\",\"req\":\"^1.0.225\"},{\"default_features\":false,\"name\":\"serde_json\",\"optional\":true,\"req\":\"^1.0.145\"},{\"features\":[\"preserve_order\"],\"kind\":\"dev\",\"name\":\"serde_json\",\"req\":\"^1.0.25\"},{\"kind\":\"dev\",\"name\":\"serde_test\",\"req\":\"^1.0.124\"},{\"name\":\"serde_with_macros\",\"optional\":true,\"req\":\"=3.15.1\"},{\"kind\":\"dev\",\"name\":\"serde_yaml\",\"req\":\"^0.9.2\"},{\"default_features\":false,\"name\":\"time_0_3\",\"optional\":true,\"package\":\"time\",\"req\":\"~0.3.36\"}],\"features\":{\"alloc\":[\"serde_core/alloc\",\"base64?/alloc\",\"chrono_0_4?/alloc\",\"hex?/alloc\",\"serde_json?/alloc\",\"time_0_3?/alloc\"],\"base64\":[\"dep:base64\",\"alloc\"],\"chrono\":[\"chrono_0_4\"],\"chrono_0_4\":[\"dep:chrono_0_4\"],\"default\":[\"std\",\"macros\"],\"guide\":[\"dep:document-features\",\"macros\",\"std\"],\"hashbrown_0_14\":[\"dep:hashbrown_0_14\",\"alloc\"],\"hashbrown_0_15\":[\"dep:hashbrown_0_15\",\"alloc\"],\"hashbrown_0_16\":[\"dep:hashbrown_0_16\",\"alloc\"],\"hex\":[\"dep:hex\",\"alloc\"],\"indexmap\":[\"indexmap_1\"],\"indexmap_1\":[\"dep:indexmap_1\",\"alloc\"],\"indexmap_2\":[\"dep:indexmap_2\",\"alloc\"],\"json\":[\"dep:serde_json\",\"alloc\"],\"macros\":[\"dep:serde_with_macros\"],\"schemars_0_8\":[\"dep:schemars_0_8\",\"std\",\"serde_with_macros?/schemars_0_8\"],\"schemars_0_9\":[\"dep:schemars_0_9\",\"alloc\",\"serde_with_macros?/schemars_0_9\",\"dep:serde_json\"],\"schemars_1\":[\"dep:schemars_1\",\"alloc\",\"serde_with_macros?/schemars_1\",\"dep:serde_json\"],\"std\":[\"alloc\",\"serde_core/std\",\"chrono_0_4?/clock\",\"chrono_0_4?/std\",\"indexmap_1?/std\",\"indexmap_2?/std\",\"time_0_3?/serde-well-known\",\"time_0_3?/std\",\"schemars_0_9?/std\",\"schemars_1?/std\"],\"time_0_3\":[\"dep:time_0_3\"]}}", - "serde_with_macros_3.15.1": "{\"dependencies\":[{\"name\":\"darling\",\"req\":\"^0.21.0\"},{\"kind\":\"dev\",\"name\":\"expect-test\",\"req\":\"^1.5.1\"},{\"kind\":\"dev\",\"name\":\"glob\",\"req\":\"^0.3.3\"},{\"kind\":\"dev\",\"name\":\"pretty_assertions\",\"req\":\"^1.4.0\"},{\"name\":\"proc-macro2\",\"req\":\"^1.0.1\"},{\"name\":\"quote\",\"req\":\"^1.0.0\"},{\"default_features\":false,\"features\":[\"std\"],\"kind\":\"dev\",\"name\":\"regex\",\"req\":\"^1.12.1\"},{\"kind\":\"dev\",\"name\":\"rustversion\",\"req\":\"^1.0.22\"},{\"features\":[\"derive\"],\"kind\":\"dev\",\"name\":\"serde\",\"req\":\"^1.0.152\"},{\"kind\":\"dev\",\"name\":\"serde_json\",\"req\":\"^1.0.25\"},{\"features\":[\"extra-traits\",\"full\",\"parsing\"],\"name\":\"syn\",\"req\":\"^2.0.0\"},{\"kind\":\"dev\",\"name\":\"trybuild\",\"req\":\"^1.0.111\"}],\"features\":{\"schemars_0_8\":[],\"schemars_0_9\":[],\"schemars_1\":[]}}", + "serde_with_3.21.0": "{\"dependencies\":[{\"default_features\":false,\"name\":\"base64\",\"optional\":true,\"req\":\"^0.22.1\"},{\"default_features\":false,\"name\":\"bs58\",\"optional\":true,\"req\":\"^0.5.1\"},{\"default_features\":false,\"features\":[\"serde\"],\"name\":\"chrono_0_4\",\"optional\":true,\"package\":\"chrono\",\"req\":\"^0.4.20\"},{\"name\":\"document-features\",\"optional\":true,\"req\":\"^0.2.7\"},{\"kind\":\"dev\",\"name\":\"expect-test\",\"req\":\"^1.5.1\"},{\"kind\":\"dev\",\"name\":\"fnv\",\"req\":\"^1.0.6\"},{\"kind\":\"dev\",\"name\":\"glob\",\"req\":\"^0.3.3\"},{\"default_features\":false,\"features\":[\"serde\"],\"name\":\"hashbrown_0_14\",\"optional\":true,\"package\":\"hashbrown\",\"req\":\"^0.14.0\"},{\"default_features\":false,\"features\":[\"serde\"],\"name\":\"hashbrown_0_15\",\"optional\":true,\"package\":\"hashbrown\",\"req\":\"^0.15.0\"},{\"default_features\":false,\"features\":[\"serde\"],\"name\":\"hashbrown_0_16\",\"optional\":true,\"package\":\"hashbrown\",\"req\":\"^0.16.0\"},{\"default_features\":false,\"features\":[\"serde\"],\"name\":\"hashbrown_0_17\",\"optional\":true,\"package\":\"hashbrown\",\"req\":\"^0.17.0\"},{\"default_features\":false,\"name\":\"hex\",\"optional\":true,\"req\":\"^0.4.3\"},{\"default_features\":false,\"features\":[\"serde-1\"],\"name\":\"indexmap_1\",\"optional\":true,\"package\":\"indexmap\",\"req\":\"^1.8\"},{\"default_features\":false,\"features\":[\"serde\"],\"name\":\"indexmap_2\",\"optional\":true,\"package\":\"indexmap\",\"req\":\"^2.0\"},{\"default_features\":false,\"features\":[\"resolve-file\"],\"kind\":\"dev\",\"name\":\"jsonschema\",\"req\":\"^0.33.0\"},{\"kind\":\"dev\",\"name\":\"mime\",\"req\":\"^0.3.16\"},{\"kind\":\"dev\",\"name\":\"pretty_assertions\",\"req\":\"^1.4.0\"},{\"default_features\":false,\"features\":[\"std\"],\"kind\":\"dev\",\"name\":\"regex\",\"req\":\"^1.12.1\"},{\"kind\":\"dev\",\"name\":\"rmp-serde\",\"req\":\"^1.3.0\"},{\"kind\":\"dev\",\"name\":\"ron\",\"req\":\"^0.12\"},{\"kind\":\"dev\",\"name\":\"rustversion\",\"req\":\"^1.0.22\"},{\"default_features\":false,\"name\":\"schemars_0_8\",\"optional\":true,\"package\":\"schemars\",\"req\":\"^0.8.16\"},{\"kind\":\"dev\",\"name\":\"schemars_0_8\",\"package\":\"schemars\",\"req\":\"^0.8.16\"},{\"default_features\":false,\"name\":\"schemars_0_9\",\"optional\":true,\"package\":\"schemars\",\"req\":\"^0.9.0\"},{\"kind\":\"dev\",\"name\":\"schemars_0_9\",\"package\":\"schemars\",\"req\":\"^0.9.0\"},{\"default_features\":false,\"name\":\"schemars_1\",\"optional\":true,\"package\":\"schemars\",\"req\":\"^1.0.2\"},{\"kind\":\"dev\",\"name\":\"schemars_1\",\"package\":\"schemars\",\"req\":\"^1.0.2\"},{\"default_features\":false,\"features\":[\"derive\"],\"kind\":\"dev\",\"name\":\"serde\",\"req\":\"^1.0.152\"},{\"kind\":\"dev\",\"name\":\"serde-xml-rs\",\"req\":\"^0.8.1\"},{\"default_features\":false,\"features\":[\"result\"],\"name\":\"serde_core\",\"req\":\"^1.0.225\"},{\"default_features\":false,\"name\":\"serde_json\",\"optional\":true,\"req\":\"^1.0.145\"},{\"features\":[\"preserve_order\"],\"kind\":\"dev\",\"name\":\"serde_json\",\"req\":\"^1.0.25\"},{\"kind\":\"dev\",\"name\":\"serde_test\",\"req\":\"^1.0.124\"},{\"name\":\"serde_with_macros\",\"optional\":true,\"req\":\"=3.21.0\"},{\"default_features\":false,\"name\":\"smallvec_1\",\"optional\":true,\"package\":\"smallvec\",\"req\":\"^1\"},{\"default_features\":false,\"name\":\"time_0_3\",\"optional\":true,\"package\":\"time\",\"req\":\"~0.3.47\"},{\"kind\":\"dev\",\"name\":\"yaml_serde\",\"req\":\"^0.10.3\"}],\"features\":{\"alloc\":[\"serde_core/alloc\",\"base64?/alloc\",\"bs58?/alloc\",\"chrono_0_4?/alloc\",\"hex?/alloc\",\"serde_json?/alloc\",\"time_0_3?/alloc\"],\"base58\":[\"dep:bs58\",\"alloc\"],\"base64\":[\"dep:base64\",\"alloc\"],\"chrono\":[\"chrono_0_4\"],\"chrono_0_4\":[\"dep:chrono_0_4\"],\"default\":[\"std\",\"macros\"],\"guide\":[\"dep:document-features\",\"macros\",\"std\"],\"hashbrown_0_14\":[\"dep:hashbrown_0_14\",\"alloc\"],\"hashbrown_0_15\":[\"dep:hashbrown_0_15\",\"alloc\"],\"hashbrown_0_16\":[\"dep:hashbrown_0_16\",\"alloc\"],\"hashbrown_0_17\":[\"dep:hashbrown_0_17\",\"alloc\"],\"hex\":[\"dep:hex\",\"alloc\"],\"indexmap\":[\"indexmap_1\"],\"indexmap_1\":[\"dep:indexmap_1\",\"alloc\"],\"indexmap_2\":[\"dep:indexmap_2\",\"alloc\"],\"json\":[\"dep:serde_json\",\"alloc\"],\"macros\":[\"dep:serde_with_macros\"],\"schemars_0_8\":[\"dep:schemars_0_8\",\"std\",\"serde_with_macros?/schemars_0_8\",\"dep:serde_json\"],\"schemars_0_9\":[\"dep:schemars_0_9\",\"alloc\",\"serde_with_macros?/schemars_0_9\",\"dep:serde_json\"],\"schemars_1\":[\"dep:schemars_1\",\"alloc\",\"serde_with_macros?/schemars_1\",\"dep:serde_json\"],\"smallvec_1\":[\"dep:smallvec_1\"],\"std\":[\"alloc\",\"bs58?/std\",\"serde_core/std\",\"chrono_0_4?/clock\",\"chrono_0_4?/std\",\"indexmap_1?/std\",\"indexmap_2?/std\",\"time_0_3?/serde-well-known\",\"time_0_3?/std\",\"schemars_0_9?/std\",\"schemars_1?/std\"],\"time_0_3\":[\"dep:time_0_3\"]}}", + "serde_with_macros_3.21.0": "{\"dependencies\":[{\"name\":\"darling\",\"req\":\"^0.23.0\"},{\"kind\":\"dev\",\"name\":\"expect-test\",\"req\":\"^1.5.1\"},{\"kind\":\"dev\",\"name\":\"glob\",\"req\":\"^0.3.3\"},{\"kind\":\"dev\",\"name\":\"pretty_assertions\",\"req\":\"^1.4.0\"},{\"name\":\"proc-macro2\",\"req\":\"^1.0.1\"},{\"name\":\"quote\",\"req\":\"^1.0.0\"},{\"default_features\":false,\"features\":[\"std\"],\"kind\":\"dev\",\"name\":\"regex\",\"req\":\"^1.12.1\"},{\"kind\":\"dev\",\"name\":\"rustversion\",\"req\":\"^1.0.22\"},{\"features\":[\"derive\"],\"kind\":\"dev\",\"name\":\"serde\",\"req\":\"^1.0.152\"},{\"kind\":\"dev\",\"name\":\"serde_json\",\"req\":\"^1.0.25\"},{\"features\":[\"extra-traits\",\"full\",\"parsing\"],\"name\":\"syn\",\"req\":\"^2.0.0\"},{\"kind\":\"dev\",\"name\":\"trybuild\",\"req\":\"^1.0.111\"}],\"features\":{\"schemars_0_8\":[],\"schemars_0_9\":[],\"schemars_1\":[]}}", "serial_test_3.5.0": "{\"dependencies\":[{\"name\":\"document-features\",\"optional\":true,\"req\":\"^0.2\"},{\"default_features\":false,\"name\":\"env_logger\",\"optional\":true,\"req\":\">=0.6.1\"},{\"default_features\":false,\"features\":[\"std\"],\"name\":\"fslock\",\"optional\":true,\"req\":\"^0.2\"},{\"default_features\":false,\"features\":[\"std\"],\"name\":\"futures-executor\",\"optional\":true,\"req\":\"^0.3\"},{\"default_features\":false,\"features\":[\"std\"],\"name\":\"futures-util\",\"optional\":true,\"req\":\"^0.3\"},{\"default_features\":false,\"features\":[\"use_std\"],\"kind\":\"dev\",\"name\":\"itertools\",\"req\":\">=0.4\"},{\"name\":\"log\",\"optional\":true,\"req\":\">=0.4.4\"},{\"default_features\":false,\"features\":[\"std\"],\"name\":\"once_cell\",\"req\":\"^1.19\"},{\"default_features\":false,\"name\":\"parking_lot\",\"req\":\"^0.12\"},{\"name\":\"serial_test_derive\",\"req\":\"~3.5.0\"}],\"features\":{\"async\":[\"dep:futures-executor\",\"dep:futures-util\",\"serial_test_derive/async\"],\"default\":[\"logging\",\"async\"],\"docsrs\":[\"dep:document-features\"],\"file_locks\":[\"dep:fslock\"],\"logging\":[\"dep:log\"],\"test_logging\":[\"logging\",\"dep:env_logger\",\"serial_test_derive/test_logging\"]}}", "serial_test_derive_3.5.0": "{\"dependencies\":[{\"default_features\":false,\"kind\":\"dev\",\"name\":\"env_logger\",\"req\":\">=0.6.1\"},{\"default_features\":false,\"kind\":\"dev\",\"name\":\"prettyplease\",\"req\":\"^0.2\"},{\"default_features\":false,\"features\":[\"proc-macro\"],\"name\":\"proc-macro2\",\"req\":\"^1.0.60\"},{\"default_features\":false,\"name\":\"quote\",\"req\":\"^1\"},{\"default_features\":false,\"features\":[\"full\",\"printing\",\"parsing\",\"clone-impls\"],\"name\":\"syn\",\"req\":\"^2\"}],\"features\":{\"async\":[],\"default\":[],\"file_locks\":[],\"test_logging\":[]}}", "sha1_0.10.6": "{\"dependencies\":[{\"name\":\"cfg-if\",\"req\":\"^1.0\"},{\"name\":\"cpufeatures\",\"req\":\"^0.2\",\"target\":\"cfg(any(target_arch = \\\"aarch64\\\", target_arch = \\\"x86\\\", target_arch = \\\"x86_64\\\"))\"},{\"name\":\"digest\",\"req\":\"^0.10.7\"},{\"features\":[\"dev\"],\"kind\":\"dev\",\"name\":\"digest\",\"req\":\"^0.10.7\"},{\"kind\":\"dev\",\"name\":\"hex-literal\",\"req\":\"^0.2.2\"},{\"name\":\"sha1-asm\",\"optional\":true,\"req\":\"^0.5\",\"target\":\"cfg(any(target_arch = \\\"aarch64\\\", target_arch = \\\"x86\\\", target_arch = \\\"x86_64\\\"))\"}],\"features\":{\"asm\":[\"sha1-asm\"],\"compress\":[],\"default\":[\"std\"],\"force-soft\":[],\"loongarch64_asm\":[],\"oid\":[\"digest/oid\"],\"std\":[\"digest/std\"]}}", @@ -1349,8 +1349,8 @@ "which_8.0.2": "{\"dependencies\":[{\"name\":\"libc\",\"optional\":true,\"req\":\"^0.2\",\"target\":\"cfg(any(unix, target_os = \\\"wasi\\\", target_os = \\\"redox\\\"))\"},{\"name\":\"regex\",\"optional\":true,\"req\":\"^1.10.2\"},{\"kind\":\"dev\",\"name\":\"tempfile\",\"req\":\"^3.9.0\"},{\"default_features\":false,\"name\":\"tracing\",\"optional\":true,\"req\":\"^0.1.40\"}],\"features\":{\"default\":[\"real-sys\"],\"real-sys\":[\"dep:libc\"],\"regex\":[\"dep:regex\"],\"tracing\":[\"dep:tracing\"]}}", "widestring_1.2.1": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"debugger_test\",\"req\":\"^0.1\"},{\"kind\":\"dev\",\"name\":\"debugger_test_parser\",\"req\":\"^0.1\"},{\"features\":[\"Win32_System_Diagnostics_Debug\"],\"kind\":\"dev\",\"name\":\"windows-sys\",\"req\":\"^0.59\"}],\"features\":{\"alloc\":[],\"debugger_visualizer\":[\"alloc\"],\"default\":[\"std\"],\"std\":[\"alloc\"]}}", "winapi-util_0.1.11": "{\"dependencies\":[{\"features\":[\"Win32_Foundation\",\"Win32_Storage_FileSystem\",\"Win32_System_Console\",\"Win32_System_SystemInformation\"],\"name\":\"windows-sys\",\"req\":\">=0.48.0, <=0.61\",\"target\":\"cfg(windows)\"}],\"features\":{}}", - "wincode-derive_0.4.5": "{\"dependencies\":[{\"name\":\"darling\",\"req\":\"^0.21.3\"},{\"name\":\"proc-macro2\",\"req\":\"^1.0.106\"},{\"name\":\"quote\",\"req\":\"^1.0.45\"},{\"features\":[\"visit-mut\",\"visit\"],\"name\":\"syn\",\"req\":\"^2.0.117\"}],\"features\":{}}", - "wincode_0.5.4": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"bincode\",\"req\":\"^1.3.3\"},{\"default_features\":false,\"name\":\"bv\",\"optional\":true,\"req\":\"^0.11.1\"},{\"features\":[\"serde\"],\"kind\":\"dev\",\"name\":\"bv\",\"req\":\"^0.11.1\"},{\"default_features\":false,\"name\":\"bytes\",\"optional\":true,\"req\":\"^1.11.1\"},{\"features\":[\"html_reports\"],\"kind\":\"dev\",\"name\":\"criterion\",\"req\":\"^0.7.0\"},{\"default_features\":false,\"name\":\"ecow\",\"optional\":true,\"req\":\"^0.2.6\"},{\"default_features\":false,\"features\":[\"serde\"],\"kind\":\"dev\",\"name\":\"ecow\",\"req\":\"^0.2.6\"},{\"default_features\":false,\"name\":\"indexmap\",\"optional\":true,\"req\":\"^2.13.1\"},{\"default_features\":false,\"features\":[\"serde\"],\"kind\":\"dev\",\"name\":\"indexmap\",\"req\":\"^2.13.1\"},{\"name\":\"pastey\",\"req\":\"^0.2.1\"},{\"kind\":\"build\",\"name\":\"proc-macro2\",\"req\":\"^1.0.106\"},{\"kind\":\"dev\",\"name\":\"proptest\",\"req\":\"^1.11.0\"},{\"kind\":\"dev\",\"name\":\"proptest-derive\",\"req\":\"^0.8.0\"},{\"kind\":\"build\",\"name\":\"quote\",\"req\":\"^1.0.45\"},{\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.9.2\"},{\"features\":[\"derive\",\"rc\"],\"kind\":\"dev\",\"name\":\"serde\",\"req\":\"^1.0.227\"},{\"default_features\":false,\"features\":[\"const_generics\"],\"name\":\"smallvec\",\"optional\":true,\"req\":\"^1.15.1\"},{\"default_features\":false,\"features\":[\"const_generics\",\"serde\"],\"kind\":\"dev\",\"name\":\"smallvec\",\"req\":\"^1.15.1\"},{\"default_features\":false,\"name\":\"solana-short-vec\",\"optional\":true,\"req\":\"^3.2.0\"},{\"kind\":\"dev\",\"name\":\"solana-short-vec\",\"req\":\"^3.2.0\"},{\"default_features\":false,\"name\":\"thiserror\",\"req\":\"^2.0.18\"},{\"default_features\":false,\"name\":\"uuid\",\"optional\":true,\"req\":\"^1.23.0\"},{\"features\":[\"serde\"],\"kind\":\"dev\",\"name\":\"uuid\",\"req\":\"^1.23.0\"},{\"name\":\"wincode-derive\",\"optional\":true,\"req\":\"^0.4.5\"}],\"features\":{\"alloc\":[],\"bv\":[\"std\",\"dep:bv\"],\"bytes\":[\"alloc\",\"dep:bytes\"],\"bytes-extra-platforms\":[\"bytes\",\"bytes/extra-platforms\"],\"default\":[\"std\"],\"derive\":[\"dep:wincode-derive\"],\"ecow\":[\"alloc\",\"dep:ecow\"],\"indexmap\":[\"alloc\",\"dep:indexmap\"],\"smallvec\":[\"alloc\",\"dep:smallvec\"],\"solana-short-vec\":[\"dep:solana-short-vec\"],\"std\":[\"alloc\",\"thiserror/std\"],\"uuid\":[\"dep:uuid\"],\"uuid-serde-compat\":[\"uuid\"]}}", + "wincode-derive_0.4.6": "{\"dependencies\":[{\"name\":\"darling\",\"req\":\"^0.23.0\"},{\"name\":\"proc-macro2\",\"req\":\"^1.0.106\"},{\"name\":\"quote\",\"req\":\"^1.0.45\"},{\"features\":[\"visit-mut\",\"visit\"],\"name\":\"syn\",\"req\":\"^2.0.117\"}],\"features\":{}}", + "wincode_0.5.5": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"bincode\",\"req\":\"^1.3.3\"},{\"default_features\":false,\"name\":\"bv\",\"optional\":true,\"req\":\"^0.11.1\"},{\"features\":[\"serde\"],\"kind\":\"dev\",\"name\":\"bv\",\"req\":\"^0.11.1\"},{\"default_features\":false,\"name\":\"bytes\",\"optional\":true,\"req\":\"^1.11.1\"},{\"features\":[\"html_reports\"],\"kind\":\"dev\",\"name\":\"criterion\",\"req\":\"^0.7.0\"},{\"default_features\":false,\"name\":\"ecow\",\"optional\":true,\"req\":\"^0.2.6\"},{\"default_features\":false,\"features\":[\"serde\"],\"kind\":\"dev\",\"name\":\"ecow\",\"req\":\"^0.2.6\"},{\"default_features\":false,\"name\":\"indexmap\",\"optional\":true,\"req\":\"^2.13.1\"},{\"default_features\":false,\"features\":[\"serde\"],\"kind\":\"dev\",\"name\":\"indexmap\",\"req\":\"^2.13.1\"},{\"name\":\"pastey\",\"req\":\"^0.2.2\"},{\"kind\":\"build\",\"name\":\"proc-macro2\",\"req\":\"^1.0.106\"},{\"kind\":\"dev\",\"name\":\"proptest\",\"req\":\"^1.11.0\"},{\"kind\":\"dev\",\"name\":\"proptest-derive\",\"req\":\"^0.8.0\"},{\"kind\":\"build\",\"name\":\"quote\",\"req\":\"^1.0.45\"},{\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.9.2\"},{\"features\":[\"derive\",\"rc\"],\"kind\":\"dev\",\"name\":\"serde\",\"req\":\"^1.0.227\"},{\"default_features\":false,\"features\":[\"const_generics\"],\"name\":\"smallvec\",\"optional\":true,\"req\":\"^1.15.1\"},{\"default_features\":false,\"features\":[\"const_generics\",\"serde\"],\"kind\":\"dev\",\"name\":\"smallvec\",\"req\":\"^1.15.1\"},{\"default_features\":false,\"name\":\"solana-short-vec\",\"optional\":true,\"req\":\"^3.2.1\"},{\"kind\":\"dev\",\"name\":\"solana-short-vec\",\"req\":\"^3.2.1\"},{\"default_features\":false,\"name\":\"thiserror\",\"req\":\"^2.0.18\"},{\"default_features\":false,\"name\":\"uuid\",\"optional\":true,\"req\":\"^1.23.1\"},{\"features\":[\"serde\"],\"kind\":\"dev\",\"name\":\"uuid\",\"req\":\"^1.23.1\"},{\"name\":\"wincode-derive\",\"optional\":true,\"req\":\"^0.4.6\"}],\"features\":{\"alloc\":[],\"bv\":[\"std\",\"dep:bv\"],\"bytes\":[\"alloc\",\"dep:bytes\"],\"bytes-extra-platforms\":[\"bytes\",\"bytes/extra-platforms\"],\"default\":[\"std\"],\"derive\":[\"dep:wincode-derive\"],\"ecow\":[\"alloc\",\"dep:ecow\"],\"indexmap\":[\"alloc\",\"dep:indexmap\"],\"smallvec\":[\"alloc\",\"dep:smallvec\"],\"solana-short-vec\":[\"dep:solana-short-vec\"],\"std\":[\"alloc\",\"thiserror/std\"],\"uuid\":[\"dep:uuid\"],\"uuid-serde-compat\":[\"uuid\"]}}", "windows-core_0.62.2": "{\"dependencies\":[{\"default_features\":false,\"name\":\"windows-implement\",\"req\":\"^0.60.2\"},{\"default_features\":false,\"name\":\"windows-interface\",\"req\":\"^0.59.3\"},{\"default_features\":false,\"name\":\"windows-link\",\"req\":\"^0.2.1\"},{\"default_features\":false,\"name\":\"windows-result\",\"req\":\"^0.4.1\"},{\"default_features\":false,\"name\":\"windows-strings\",\"req\":\"^0.5.1\"}],\"features\":{\"default\":[\"std\"],\"std\":[\"windows-result/std\",\"windows-strings/std\"]}}", "windows-implement_0.60.2": "{\"dependencies\":[{\"default_features\":false,\"name\":\"proc-macro2\",\"req\":\"^1.0\"},{\"default_features\":false,\"name\":\"quote\",\"req\":\"^1.0\"},{\"default_features\":false,\"features\":[\"parsing\",\"proc-macro\",\"printing\",\"full\",\"clone-impls\"],\"name\":\"syn\",\"req\":\"^2.0\"}],\"features\":{}}", "windows-interface_0.59.3": "{\"dependencies\":[{\"default_features\":false,\"name\":\"proc-macro2\",\"req\":\"^1.0\"},{\"default_features\":false,\"name\":\"quote\",\"req\":\"^1.0\"},{\"default_features\":false,\"features\":[\"parsing\",\"proc-macro\",\"printing\",\"full\",\"clone-impls\"],\"name\":\"syn\",\"req\":\"^2.0\"}],\"features\":{}}", From 8a8be1879d70eceff4caf1b73d6ae63c61fddc5d Mon Sep 17 00:00:00 2001 From: Tom Parker-Shemilt Date: Thu, 16 Jul 2026 21:55:56 +0100 Subject: [PATCH 11/84] Remove more unused items (#2561) --- .github/actions/free-disk/action.yaml | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/.github/actions/free-disk/action.yaml b/.github/actions/free-disk/action.yaml index 2bf00b955..77e405e4b 100644 --- a/.github/actions/free-disk/action.yaml +++ b/.github/actions/free-disk/action.yaml @@ -13,18 +13,26 @@ runs: endersonmenezes/free-disk-space@7901478139cff6e9d44df5972fd8ab8fcade4db1 with: rm_cmd: "rmz" # For speed up + rmz_version: "3.2.0" remove_android: false # Takes too long. remove_dotnet: true remove_haskell: true - remove_tool_cache: false # TODO(palfrey): Do we really need this? - # Note: Not deleting google-cloud-cli because it takes too long. + remove_tool_cache: false remove_folders: > + /opt/go + /opt/Ruby + /opt/microsoft/msedge + /usr/local/lib/android /usr/share/swift /usr/share/miniconda /usr/share/az* /usr/share/glade* /usr/local/share/chromium /usr/local/share/powershell + /usr/local/julia + /usr/local/aws-cli + /usr/local/aws-sam-cli + /usr/share/gradle # using hints from https://github.com/actions/runner-images/issues/10511#issuecomment-3984466720 - if: runner.os == 'macOS' From 65e2ec2004ca28e3d9273959c7a809f9b0dcb0e0 Mon Sep 17 00:00:00 2001 From: Tom Parker-Shemilt Date: Thu, 16 Jul 2026 22:53:43 +0100 Subject: [PATCH 12/84] Remove quoting around NetApp (#2556) * Remove quoting around NetApp * Fix clippy permitted list of idents --- clippy.toml | 79 +++++++++++++++++++ nativelink-config/src/stores.rs | 4 +- tools/generate-bazel-rc/src/main.rs | 2 +- .../reference/nativelink-config/index.mdx | 4 +- .../docs/reference/nativelink-config/main.mdx | 4 +- .../reference/nativelink-config/v1.0.0.mdx | 4 +- .../reference/nativelink-config/v1.1.0.mdx | 4 +- .../reference/nativelink-config/v1.2.0.mdx | 4 +- .../reference/nativelink-config/v1.3.0.mdx | 4 +- .../reference/nativelink-config/v1.3.1.mdx | 4 +- .../reference/nativelink-config/v1.3.2.mdx | 4 +- .../reference/nativelink-config/v1.4.0.mdx | 4 +- .../reference/nativelink-config/v1.5.0.mdx | 4 +- .../reference/nativelink-config/v1.5.1.mdx | 4 +- .../reference/nativelink-config/v1.5.2.mdx | 4 +- .../reference/nativelink-config/v1.6.0.mdx | 4 +- web/apps/docs/scripts/lib/schema-to-mdx.mjs | 5 ++ 17 files changed, 113 insertions(+), 29 deletions(-) diff --git a/clippy.toml b/clippy.toml index 925f27f2b..2e3ed029d 100644 --- a/clippy.toml +++ b/clippy.toml @@ -14,4 +14,83 @@ disallowed-methods = [ { path = "tokio::task::spawn_blocking", reason = "use `nativelink-util::task::spawn_blocking` instead" }, { path = "tokio::task::spawn_local", reason = "use one of the `nativelink-util::task` functions instead" }, ] +# Started from the default at https://doc.rust-lang.org/clippy/lint_configuration.html#doc-valid-idents +doc-valid-idents = [ + "AccessKit", + "BibLaTeX", + "BibTeX", + "CamelCase", + "ClojureScript", + "CoAP", + "CoffeeScript", + "CoreFoundation", + "CoreGraphics", + "CoreText", + "DevOps", + "Direct2D", + "Direct3D", + "DirectWrite", + "DirectX", + "ECMAScript", + "EiB", + "FreeBSD", + "GHz", + "GPLv2", + "GPLv3", + "GiB", + "GitHub", + "GitLab", + "GraphQL", + "IPv4", + "IPv6", + "InfiniBand", + "JavaScript", + "KiB", + "LaTeX", + "MHz", + "MiB", + "MinGW", + "NaN", + "NaNs", + "NetApp", + "NetBSD", + "NixOS", + "OAuth", + "OCaml", + "OpenAL", + "OpenBSD", + "OpenDNS", + "OpenExr", + "OpenGL", + "OpenMP", + "OpenSSH", + "OpenSSL", + "OpenStreetMap", + "OpenTelemetry", + "OpenType", + "PiB", + "PostScript", + "PowerPC", + "PowerShell", + "PureScript", + "RoCE", + "THz", + "TeX", + "TensorFlow", + "TiB", + "TrueType", + "TypeScript", + "WebAssembly", + "WebGL", + "WebGL2", + "WebGPU", + "WebP", + "WebRTC", + "WebSocket", + "WebTransport", + "YCbCr", + "iOS", + "macOS", + "sRGB", +] msrv = "1.93.1" diff --git a/nativelink-config/src/stores.rs b/nativelink-config/src/stores.rs index c97c1fceb..3d3469861 100644 --- a/nativelink-config/src/stores.rs +++ b/nativelink-config/src/stores.rs @@ -153,8 +153,8 @@ pub enum StoreSpec { /// } /// ``` /// - /// 4. **`NetApp` ONTAP S3** - /// `NetApp` ONTAP S3 store will use ONTAP's S3-compatible storage as a backend + /// 4. **NetApp ONTAP S3:** + /// NetApp ONTAP S3 store will use ONTAP's S3-compatible storage as a backend /// to store files. This store is specifically configured for ONTAP's S3 requirements /// including custom TLS configuration, credentials management, and proper vserver /// configuration. diff --git a/tools/generate-bazel-rc/src/main.rs b/tools/generate-bazel-rc/src/main.rs index a0d54021b..1debc5285 100644 --- a/tools/generate-bazel-rc/src/main.rs +++ b/tools/generate-bazel-rc/src/main.rs @@ -76,7 +76,7 @@ fn get_lints_from_key(lints_table: &Map, key: &str) -> BTreeSet Date: Fri, 17 Jul 2026 08:50:34 +0100 Subject: [PATCH 13/84] Upgrade quick-xml to 0.41 for RUSTSEC-2026-0194 (#2566) Fixes #2564 --- Cargo.lock | 8 ++++---- MODULE.bazel.lock | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 2b405f954..f968b6792 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3928,9 +3928,9 @@ dependencies = [ [[package]] name = "quick-xml" -version = "0.39.4" +version = "0.41.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cdcc8dd4e2f670d309a5f0e83fe36dfdc05af317008fea29144da1a2ac858e5e" +checksum = "e660451e55124f798a69a5af3f49ccfbefbd41910eefd25caf2393e1f3473ec1" dependencies = [ "memchr", "serde", @@ -5451,9 +5451,9 @@ checksum = "562d481066bde0658276a35467c4af00bdc6ee726305698a55b86e61d7ad82bb" [[package]] name = "typespec" -version = "1.0.0" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "21666a31293beab8f41d38c2849ddbc342cd9c7cb4d71a9818868287a8934e53" +checksum = "753a2fe021e407d4fc9ee6f4f0a33403cc306d5c54c4e4ebe1b8cbde0ca052b9" dependencies = [ "base64 0.22.1", "bytes", diff --git a/MODULE.bazel.lock b/MODULE.bazel.lock index c1eef96bc..8fb6a7f22 100644 --- a/MODULE.bazel.lock +++ b/MODULE.bazel.lock @@ -1146,7 +1146,7 @@ "pyo3-macros-backend_0.28.3": "{\"dependencies\":[{\"name\":\"heck\",\"req\":\"^0.5\"},{\"default_features\":false,\"name\":\"proc-macro2\",\"req\":\"^1.0.60\"},{\"features\":[\"resolve-config\"],\"name\":\"pyo3-build-config\",\"req\":\"=0.28.3\"},{\"kind\":\"build\",\"name\":\"pyo3-build-config\",\"req\":\"=0.28.3\"},{\"default_features\":false,\"name\":\"quote\",\"req\":\"^1.0.37\"},{\"default_features\":false,\"features\":[\"derive\",\"parsing\",\"printing\",\"clone-impls\",\"full\",\"extra-traits\",\"visit-mut\"],\"name\":\"syn\",\"req\":\"^2.0.59\"}],\"features\":{\"experimental-async\":[],\"experimental-inspect\":[]}}", "pyo3-macros_0.28.3": "{\"dependencies\":[{\"default_features\":false,\"name\":\"proc-macro2\",\"req\":\"^1.0.60\"},{\"name\":\"pyo3-macros-backend\",\"req\":\"=0.28.3\"},{\"name\":\"quote\",\"req\":\"^1\"},{\"features\":[\"full\",\"extra-traits\"],\"name\":\"syn\",\"req\":\"^2\"}],\"features\":{\"experimental-async\":[\"pyo3-macros-backend/experimental-async\"],\"experimental-inspect\":[\"pyo3-macros-backend/experimental-inspect\"],\"multiple-pymethods\":[]}}", "pyo3_0.28.3": "{\"dependencies\":[{\"name\":\"anyhow\",\"optional\":true,\"req\":\"^1.0.1\"},{\"kind\":\"dev\",\"name\":\"assert_approx_eq\",\"req\":\"^1.1.0\"},{\"name\":\"bigdecimal\",\"optional\":true,\"req\":\"^0.4.7\"},{\"name\":\"bytes\",\"optional\":true,\"req\":\"^1.10\"},{\"default_features\":false,\"name\":\"chrono\",\"optional\":true,\"req\":\"^0.4.25\"},{\"kind\":\"dev\",\"name\":\"chrono\",\"req\":\"^0.4.25\"},{\"default_features\":false,\"name\":\"chrono-tz\",\"optional\":true,\"req\":\">=0.10, <0.11\"},{\"kind\":\"dev\",\"name\":\"chrono-tz\",\"req\":\">=0.10, <0.11\"},{\"name\":\"either\",\"optional\":true,\"req\":\"^1.9\"},{\"name\":\"eyre\",\"optional\":true,\"req\":\">=0.6.8, <0.7\"},{\"kind\":\"dev\",\"name\":\"futures\",\"req\":\"^0.3.28\"},{\"default_features\":false,\"name\":\"hashbrown\",\"optional\":true,\"req\":\">=0.15.0, <0.17\"},{\"features\":[\"fallback\"],\"name\":\"iana-time-zone\",\"optional\":true,\"req\":\"^0.1\"},{\"name\":\"indexmap\",\"optional\":true,\"req\":\">=2.5.0, <3\"},{\"name\":\"inventory\",\"optional\":true,\"req\":\"^0.3.5\"},{\"name\":\"jiff-02\",\"optional\":true,\"package\":\"jiff\",\"req\":\"^0.2\"},{\"name\":\"libc\",\"req\":\"^0.2.62\"},{\"name\":\"lock_api\",\"optional\":true,\"req\":\"^0.4\"},{\"name\":\"num-bigint\",\"optional\":true,\"req\":\"^0.4.4\"},{\"name\":\"num-complex\",\"optional\":true,\"req\":\">=0.4.6, <0.5\"},{\"name\":\"num-rational\",\"optional\":true,\"req\":\"^0.4.1\"},{\"name\":\"num-traits\",\"optional\":true,\"req\":\"^0.2.16\"},{\"name\":\"once_cell\",\"req\":\"^1.21\"},{\"default_features\":false,\"name\":\"ordered-float\",\"optional\":true,\"req\":\"^5.0.0\"},{\"name\":\"parking_lot\",\"optional\":true,\"req\":\"^0.12\"},{\"features\":[\"arc_lock\"],\"kind\":\"dev\",\"name\":\"parking_lot\",\"req\":\"^0.12.3\"},{\"name\":\"portable-atomic\",\"req\":\"^1.0\",\"target\":\"cfg(not(target_has_atomic = \\\"64\\\"))\"},{\"default_features\":false,\"features\":[\"std\"],\"kind\":\"dev\",\"name\":\"proptest\",\"req\":\"^1.0\"},{\"features\":[\"resolve-config\"],\"kind\":\"build\",\"name\":\"pyo3-build-config\",\"req\":\"=0.28.3\"},{\"name\":\"pyo3-ffi\",\"req\":\"=0.28.3\"},{\"name\":\"pyo3-macros\",\"optional\":true,\"req\":\"=0.28.3\"},{\"kind\":\"dev\",\"name\":\"rayon\",\"req\":\"^1.6.1\"},{\"default_features\":false,\"name\":\"rust_decimal\",\"optional\":true,\"req\":\"^1.15\"},{\"kind\":\"dev\",\"name\":\"send_wrapper\",\"req\":\"^0.6\"},{\"name\":\"serde\",\"optional\":true,\"req\":\"^1.0\"},{\"features\":[\"derive\"],\"kind\":\"dev\",\"name\":\"serde\",\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"serde_json\",\"req\":\"^1.0.61\"},{\"name\":\"smallvec\",\"optional\":true,\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"static_assertions\",\"req\":\"^1.1.0\"},{\"kind\":\"dev\",\"name\":\"tempfile\",\"req\":\"^3.12.0\"},{\"default_features\":false,\"name\":\"time\",\"optional\":true,\"req\":\"^0.3.38\"},{\"kind\":\"dev\",\"name\":\"trybuild\",\"req\":\">=1.0.115\"},{\"name\":\"uuid\",\"optional\":true,\"req\":\"^1.12.0\"},{\"features\":[\"v4\"],\"kind\":\"dev\",\"name\":\"uuid\",\"req\":\"^1.10.0\"}],\"features\":{\"abi3\":[\"pyo3-build-config/abi3\",\"pyo3-ffi/abi3\"],\"abi3-py310\":[\"abi3-py311\",\"pyo3-build-config/abi3-py310\",\"pyo3-ffi/abi3-py310\"],\"abi3-py311\":[\"abi3-py312\",\"pyo3-build-config/abi3-py311\",\"pyo3-ffi/abi3-py311\"],\"abi3-py312\":[\"abi3-py313\",\"pyo3-build-config/abi3-py312\",\"pyo3-ffi/abi3-py312\"],\"abi3-py313\":[\"abi3-py314\",\"pyo3-build-config/abi3-py313\",\"pyo3-ffi/abi3-py313\"],\"abi3-py314\":[\"abi3\",\"pyo3-build-config/abi3-py314\",\"pyo3-ffi/abi3-py314\"],\"abi3-py37\":[\"abi3-py38\",\"pyo3-build-config/abi3-py37\",\"pyo3-ffi/abi3-py37\"],\"abi3-py38\":[\"abi3-py39\",\"pyo3-build-config/abi3-py38\",\"pyo3-ffi/abi3-py38\"],\"abi3-py39\":[\"abi3-py310\",\"pyo3-build-config/abi3-py39\",\"pyo3-ffi/abi3-py39\"],\"arc_lock\":[\"lock_api\",\"lock_api/arc_lock\",\"parking_lot?/arc_lock\"],\"auto-initialize\":[],\"bigdecimal\":[\"dep:bigdecimal\",\"num-bigint\"],\"chrono-local\":[\"chrono/clock\",\"dep:iana-time-zone\"],\"default\":[\"macros\"],\"experimental-async\":[\"macros\",\"pyo3-macros/experimental-async\"],\"experimental-inspect\":[\"pyo3-macros/experimental-inspect\"],\"extension-module\":[\"pyo3-ffi/extension-module\"],\"full\":[\"macros\",\"anyhow\",\"arc_lock\",\"bigdecimal\",\"bytes\",\"chrono\",\"chrono-local\",\"chrono-tz\",\"either\",\"experimental-async\",\"experimental-inspect\",\"eyre\",\"hashbrown\",\"indexmap\",\"jiff-02\",\"lock_api\",\"num-bigint\",\"num-complex\",\"num-rational\",\"ordered-float\",\"parking_lot\",\"py-clone\",\"rust_decimal\",\"serde\",\"smallvec\",\"time\",\"uuid\"],\"generate-import-lib\":[\"pyo3-ffi/generate-import-lib\"],\"macros\":[\"pyo3-macros\"],\"multiple-pymethods\":[\"inventory\",\"pyo3-macros/multiple-pymethods\"],\"nightly\":[],\"num-bigint\":[\"dep:num-bigint\",\"dep:num-traits\"],\"parking_lot\":[\"dep:parking_lot\",\"lock_api\"],\"py-clone\":[]}}", - "quick-xml_0.39.4": "{\"dependencies\":[{\"features\":[\"derive\"],\"name\":\"arbitrary\",\"optional\":true,\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"criterion\",\"req\":\">=0.4, <0.9\"},{\"name\":\"document-features\",\"optional\":true,\"req\":\"^0.2\"},{\"name\":\"encoding_rs\",\"optional\":true,\"req\":\"^0.8\"},{\"name\":\"memchr\",\"req\":\"^2.1\"},{\"kind\":\"dev\",\"name\":\"pretty_assertions\",\"req\":\"^1.4\"},{\"kind\":\"dev\",\"name\":\"regex\",\"req\":\"^1\"},{\"name\":\"serde\",\"optional\":true,\"req\":\">=1.0.180\"},{\"kind\":\"dev\",\"name\":\"serde-value\",\"req\":\"^0.7\"},{\"kind\":\"dev\",\"name\":\"serde_derive\",\"req\":\"^1.0.206\"},{\"default_features\":false,\"features\":[\"io-util\"],\"name\":\"tokio\",\"optional\":true,\"req\":\"^1.10\"},{\"default_features\":false,\"features\":[\"macros\",\"rt\"],\"kind\":\"dev\",\"name\":\"tokio\",\"req\":\"^1.21\"},{\"kind\":\"dev\",\"name\":\"tokio-test\",\"req\":\"^0.4\"}],\"features\":{\"async-tokio\":[\"tokio\"],\"default\":[],\"encoding\":[\"encoding_rs\"],\"escape-html\":[],\"overlapped-lists\":[],\"serde-types\":[\"serde/derive\"],\"serialize\":[\"serde\"]}}", + "quick-xml_0.41.0": "{\"dependencies\":[{\"features\":[\"derive\"],\"name\":\"arbitrary\",\"optional\":true,\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"criterion\",\"req\":\">=0.4, <0.9\"},{\"name\":\"document-features\",\"optional\":true,\"req\":\"^0.2\"},{\"name\":\"encoding_rs\",\"optional\":true,\"req\":\"^0.8\"},{\"name\":\"memchr\",\"req\":\"^2.1\"},{\"kind\":\"dev\",\"name\":\"pretty_assertions\",\"req\":\"^1.4\"},{\"kind\":\"dev\",\"name\":\"regex\",\"req\":\"^1\"},{\"name\":\"serde\",\"optional\":true,\"req\":\">=1.0.180\"},{\"kind\":\"dev\",\"name\":\"serde-value\",\"req\":\"^0.7\"},{\"kind\":\"dev\",\"name\":\"serde_derive\",\"req\":\"^1.0.206\"},{\"default_features\":false,\"features\":[\"io-util\"],\"name\":\"tokio\",\"optional\":true,\"req\":\"^1.10\"},{\"default_features\":false,\"features\":[\"macros\",\"rt\"],\"kind\":\"dev\",\"name\":\"tokio\",\"req\":\"^1.21\"},{\"kind\":\"dev\",\"name\":\"tokio-test\",\"req\":\"^0.4\"}],\"features\":{\"async-tokio\":[\"tokio\"],\"default\":[],\"encoding\":[\"encoding_rs\"],\"escape-html\":[],\"overlapped-lists\":[],\"serde-types\":[\"serde/derive\"],\"serialize\":[\"serde\"]}}", "quote_1.0.45": "{\"dependencies\":[{\"default_features\":false,\"name\":\"proc-macro2\",\"req\":\"^1.0.80\"},{\"kind\":\"dev\",\"name\":\"rustversion\",\"req\":\"^1.0\"},{\"features\":[\"diff\"],\"kind\":\"dev\",\"name\":\"trybuild\",\"req\":\"^1.0.108\"}],\"features\":{\"default\":[\"proc-macro\"],\"proc-macro\":[\"proc-macro2/proc-macro\"]}}", "r-efi_5.3.0": "{\"dependencies\":[{\"name\":\"core\",\"optional\":true,\"package\":\"rustc-std-workspace-core\",\"req\":\"^1.0.0\"}],\"features\":{\"efiapi\":[],\"examples\":[\"native\"],\"native\":[],\"rustc-dep-of-std\":[\"core\"]}}", "r-efi_6.0.0": "{\"dependencies\":[{\"name\":\"core\",\"optional\":true,\"package\":\"rustc-std-workspace-core\",\"req\":\"^1.0.0\"}],\"features\":{\"native\":[],\"rustc-dep-of-std\":[\"core\"]}}", @@ -1306,7 +1306,7 @@ "typed-builder_0.20.1": "{\"dependencies\":[{\"name\":\"typed-builder-macro\",\"req\":\"=0.20.1\"}],\"features\":{}}", "typed-path_0.12.3": "{\"dependencies\":[],\"features\":{\"default\":[\"std\"],\"std\":[]}}", "typenum_1.19.0": "{\"dependencies\":[{\"default_features\":false,\"name\":\"scale-info\",\"optional\":true,\"req\":\"^1.0\"}],\"features\":{\"const-generics\":[],\"force_unix_path_separator\":[],\"i128\":[],\"no_std\":[],\"scale_info\":[\"scale-info/derive\"],\"strict\":[]}}", - "typespec_1.0.0": "{\"dependencies\":[{\"name\":\"base64\",\"req\":\"^0.22\"},{\"name\":\"bytes\",\"req\":\"^1.11.1\"},{\"name\":\"futures\",\"req\":\"^0.3\"},{\"features\":[\"serialize\",\"serde-types\"],\"name\":\"quick-xml\",\"optional\":true,\"req\":\"^0.39.0\"},{\"features\":[\"derive\"],\"name\":\"serde\",\"optional\":true,\"req\":\"^1.0\"},{\"name\":\"serde_json\",\"optional\":true,\"req\":\"^1.0.149\"},{\"kind\":\"dev\",\"name\":\"thiserror\",\"req\":\"^2.0\"},{\"name\":\"url\",\"req\":\"^2.5\"}],\"features\":{\"default\":[\"http\",\"json\"],\"http\":[],\"json\":[\"dep:serde\",\"dep:serde_json\"],\"xml\":[\"dep:serde\",\"dep:quick-xml\"]}}", + "typespec_1.1.0": "{\"dependencies\":[{\"name\":\"base64\",\"req\":\"^0.22\"},{\"name\":\"bytes\",\"req\":\"^1.11.1\"},{\"name\":\"futures\",\"req\":\"^0.3\"},{\"features\":[\"serialize\",\"serde-types\"],\"name\":\"quick-xml\",\"optional\":true,\"req\":\"^0.41.0\"},{\"features\":[\"derive\"],\"name\":\"serde\",\"optional\":true,\"req\":\"^1.0\"},{\"name\":\"serde_json\",\"optional\":true,\"req\":\"^1.0.149\"},{\"kind\":\"dev\",\"name\":\"thiserror\",\"req\":\"^2.0\"},{\"name\":\"url\",\"req\":\"^2.5\"}],\"features\":{\"default\":[\"http\",\"json\"],\"http\":[],\"json\":[\"dep:serde\",\"dep:serde_json\"],\"xml\":[\"dep:serde\",\"dep:quick-xml\"]}}", "typespec_client_core_1.0.0": "{\"dependencies\":[{\"name\":\"async-trait\",\"req\":\"^0.1\"},{\"name\":\"base64\",\"req\":\"^0.22\"},{\"name\":\"bytes\",\"req\":\"^1.11.1\"},{\"name\":\"dyn-clone\",\"req\":\"^1.0\"},{\"name\":\"futures\",\"req\":\"^0.3\"},{\"name\":\"gloo-timers\",\"optional\":true,\"req\":\"^0.3\"},{\"name\":\"pin-project\",\"req\":\"^1.1\"},{\"features\":[\"sys_rng\"],\"name\":\"rand\",\"req\":\"^0.10.1\"},{\"default_features\":false,\"features\":[\"stream\"],\"name\":\"reqwest\",\"optional\":true,\"req\":\"^0.13.2\"},{\"name\":\"rust_decimal\",\"optional\":true,\"req\":\"^1.40.0\"},{\"features\":[\"derive\"],\"name\":\"serde\",\"req\":\"^1.0\"},{\"name\":\"serde_json\",\"optional\":true,\"req\":\"^1.0.149\"},{\"features\":[\"serde-well-known\",\"macros\"],\"name\":\"time\",\"req\":\"^0.3.47\"},{\"default_features\":false,\"features\":[\"macros\",\"time\",\"macros\",\"rt-multi-thread\",\"time\"],\"name\":\"tokio\",\"optional\":true,\"req\":\"^1.49\"},{\"default_features\":false,\"features\":[\"macros\",\"time\",\"fs\"],\"kind\":\"dev\",\"name\":\"tokio\",\"req\":\"^1.49\"},{\"name\":\"tracing\",\"req\":\"^0.1.44\"},{\"kind\":\"dev\",\"name\":\"tracing\",\"req\":\"^0.1.44\"},{\"kind\":\"dev\",\"name\":\"tracing-subscriber\",\"req\":\"^0.3\"},{\"default_features\":false,\"name\":\"typespec\",\"req\":\"^1.0.0\"},{\"name\":\"typespec_macros\",\"optional\":true,\"req\":\"^1.0.0\"},{\"kind\":\"dev\",\"name\":\"typespec_macros\",\"req\":\"^1.0.0\"},{\"name\":\"url\",\"req\":\"^2.5\"},{\"features\":[\"v4\"],\"name\":\"uuid\",\"req\":\"^1.20\"}],\"features\":{\"debug\":[\"typespec_macros?/debug\"],\"decimal\":[\"dep:rust_decimal\"],\"default\":[\"http\",\"json\",\"reqwest\",\"reqwest_deflate\",\"reqwest_gzip\",\"reqwest_rustls\",\"tokio\"],\"derive\":[\"dep:typespec_macros\"],\"http\":[\"typespec/http\"],\"json\":[\"dep:serde_json\",\"typespec/json\"],\"reqwest\":[\"dep:reqwest\"],\"reqwest_deflate\":[\"reqwest\",\"reqwest/deflate\"],\"reqwest_gzip\":[\"reqwest\",\"reqwest/gzip\"],\"reqwest_rustls\":[\"reqwest\",\"reqwest/rustls\"],\"test\":[],\"tokio\":[\"tokio/sync\",\"tokio/time\"],\"xml\":[\"dep:serde_json\",\"typespec/xml\"]}}", "typespec_macros_1.0.0": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"cargo_metadata\",\"req\":\"^0.23.1\"},{\"name\":\"proc-macro2\",\"req\":\"^1.0.106\"},{\"name\":\"quote\",\"req\":\"^1.0.44\"},{\"name\":\"rustc_version\",\"req\":\"^0.4\"},{\"features\":[\"derive\"],\"kind\":\"dev\",\"name\":\"serde\",\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"serde_json\",\"req\":\"^1.0.149\"},{\"features\":[\"full\"],\"name\":\"syn\",\"req\":\"^2.0.115\"},{\"default_features\":false,\"features\":[\"macros\",\"time\"],\"kind\":\"dev\",\"name\":\"tokio\",\"req\":\"^1.49\"}],\"features\":{\"debug\":[]}}", "ucd-trie_0.1.7": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"once_cell\",\"req\":\"^1\"}],\"features\":{\"default\":[\"std\"],\"std\":[]}}", From 2a4d961782a6944a9a29f06a20b517821221dc1f Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Fri, 17 Jul 2026 09:29:14 +0100 Subject: [PATCH 14/84] docs(config-reference): regenerate for NativeLink (#2568) Co-authored-by: config reference bot --- .../docs/content/docs/reference/nativelink-config/main.mdx | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/web/apps/docs/content/docs/reference/nativelink-config/main.mdx b/web/apps/docs/content/docs/reference/nativelink-config/main.mdx index 34baaf36d..330ea9482 100644 --- a/web/apps/docs/content/docs/reference/nativelink-config/main.mdx +++ b/web/apps/docs/content/docs/reference/nativelink-config/main.mdx @@ -5,7 +5,7 @@ full: true --- {/* AUTOGENERATED — do not edit by hand. - Source: nativelink-config @ main (4be35e6d) + Source: nativelink-config @ main (65e2ec20) Regenerate from web/: bun --filter @nativelink/docs gen:config-reference */} @@ -937,6 +937,7 @@ Configuration for `ExperimentalMongoDB` store. | `allocation_strategy` | [WorkerAllocationStrategy](#workerallocationstrategy) | — | `"least_recently_used"` | The strategy used to assign workers jobs. | | `experimental_backend` | [ExperimentalSimpleSchedulerBackend](#experimentalsimpleschedulerbackend) | — | memory | The storage backend to use for the scheduler. | | `worker_match_logging_interval_s` | integer (int64) | — | `10` | Every N seconds, do logging of worker matching e.g. "worker busy", "can't find any worker" Defaults to 10s. Can be set to `-1` to disable | +| `fallback_match_interval_s` | integer (int64) | — | `5` | Every N seconds, run a worker matching pass even if no task or worker change notification arrived. This is a safety net for missed notifications and for scheduler backends with eventually consistent searches (for example Redis), where an operation that was re-queued may not be visible to the search triggered by its own notification. Without this, such an operation can stay queued until an unrelated event triggers another matching pass. Defaults to 5s. Zero or any negative value disables it. | ## SchedulerGrpcSpec From 6675acb9ed130ee11f174772a0677500944c8049 Mon Sep 17 00:00:00 2001 From: Marcus Eagan Date: Fri, 17 Jul 2026 10:51:40 +0100 Subject: [PATCH 15/84] Upgrade anyhow to 1.0.103 for RUSTSEC-2026-0190 (#2569) --- nativelink-util/Cargo.toml | 2 +- tools/cargo-llvm-cov/Cargo.lock | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/nativelink-util/Cargo.toml b/nativelink-util/Cargo.toml index 09cdedf47..0c75df3be 100644 --- a/nativelink-util/Cargo.toml +++ b/nativelink-util/Cargo.toml @@ -98,7 +98,7 @@ walkdir = { version = "2.5.0", default-features = false } wincode = { version = "0.5.4", default-features = false, features = ["derive"] } [dev-dependencies] -anyhow = { version = "1.0.100", default-features = false } +anyhow = { version = "1.0.103", default-features = false } nativelink-macro = { path = "../nativelink-macro" } opentelemetry-proto = { version = "0.32.0", default-features = false, features = [ "gen-tonic", diff --git a/tools/cargo-llvm-cov/Cargo.lock b/tools/cargo-llvm-cov/Cargo.lock index 40fa992bb..c8499e41a 100644 --- a/tools/cargo-llvm-cov/Cargo.lock +++ b/tools/cargo-llvm-cov/Cargo.lock @@ -13,9 +13,9 @@ dependencies = [ [[package]] name = "anyhow" -version = "1.0.102" +version = "1.0.103" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f202df86484c868dbad7eaa557ef785d5c66295e41b460ef922eca0723b842c" +checksum = "2a4385e2e34eb35d6b3efe798b9eb88096925d87726c0798709bf56d9ed84af3" [[package]] name = "autocfg" From b01cb0db985a8c5356b58751460981c97bfffd6f Mon Sep 17 00:00:00 2001 From: Marcus Eagan Date: Fri, 17 Jul 2026 15:15:59 +0100 Subject: [PATCH 16/84] Fix AsyncFixedBuffer with zstd compression, remove unused BufChannelReader adapter (#2574) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * test(service): compressed-download encode streams must not starve the blocking pool A compressed-download encode stream drains at the gRPC client's pace. The current implementation holds one tokio blocking-pool thread per stream for the stream's entire lifetime (BufChannelReader blocks waiting for input and blocking_send parks until the client consumes), so N concurrent downloads with slow consumers occupy N pool threads and every unrelated spawn_blocking user queues behind them. This test wires encode streams the way ByteStreamServer::inner_read_compressed does, holds more of them than the pool has threads, and asserts a trivial unrelated spawn_blocking closure still runs. It fails against the current thread-per-stream implementation. Co-Authored-By: Claude Opus 4.8 (1M context) * fix(service): encode compressed downloads asynchronously, off the blocking pool stream_encode_compressed_download previously ran under spawn_blocking with a blocking reader over raw_rx and blocking_send into the output channel, so one tokio blocking-pool thread was parked per compressed download for the whole stream at the client's drain rate. Concurrent downloads with slow consumers could exhaust the pool (default cap 512), starving every other spawn_blocking user — filesystem store I/O, upload decode, credential resolution — and the per-thread zstd contexts, stacks, and buffers inflated memory accordingly. The encoder is now an async fn driving zstd's raw streaming API (ZSTD_compressStream via zstd::stream::raw::Encoder): async recv, inline per-chunk encode (CPU bounded by the small channel chunk size), async send for backpressure. No thread is held while waiting on either channel. The wire format is unchanged: one well-formed zstd frame, identical to what zstd::stream::read::Encoder emitted. The bytestream_server caller drops its spawn_blocking wrapper; dropping the read stream still cancels the encode because the future itself is dropped. Co-Authored-By: Claude Opus 4.8 (1M context) * fix(service): decode compressed uploads asynchronously, off the blocking pool Same defect shape as the download encode path: stream_decode_compressed_upload ran under spawn_blocking, blocking on compressed_rx for client bytes and parking in blocking_send while the store drained, holding one blocking-pool thread per compressed upload for the stream's lifetime. The decoder is now an async fn driving zstd's raw streaming API (ZSTD_decompressStream via zstd::stream::raw::Decoder) with async channel sends. Validation is unchanged: the per-chunk decoded-size cap (bomb rejection), the exact final-size check, and the digest check are preserved, and a stream that ends mid-frame is rejected as InvalidArgument (previously surfaced as a read error from the blocking decoder). Co-Authored-By: Claude Opus 4.8 (1M context) * refactor(service): remove unused BufChannelReader adapter The async streaming zstd rewrite left this blocking Read adapter with no production users; drop it and its unit test. Co-Authored-By: Claude Fable 5 --------- Co-authored-by: Walter Gray Co-authored-by: Claude Opus 4.8 (1M context) --- nativelink-service/src/bytestream_server.rs | 46 ++-- nativelink-service/src/wire_compression.rs | 215 +++++++++++------- .../tests/wire_compression_test.rs | 140 +++++++++--- 3 files changed, 254 insertions(+), 147 deletions(-) diff --git a/nativelink-service/src/bytestream_server.rs b/nativelink-service/src/bytestream_server.rs index 026a511e0..c678eed59 100644 --- a/nativelink-service/src/bytestream_server.rs +++ b/nativelink-service/src/bytestream_server.rs @@ -50,9 +50,9 @@ use nativelink_util::digest_hasher::{ }; use nativelink_util::proto_stream_utils::WriteRequestStreamWrapper; use nativelink_util::resource_info::ResourceInfo; +use nativelink_util::spawn; use nativelink_util::store_trait::{Store, StoreLike, StoreOptimizations, UploadSizeInfo}; use nativelink_util::task::JoinHandleDropGuard; -use nativelink_util::{spawn, spawn_blocking}; use opentelemetry::context::FutureExt; use parking_lot::Mutex; use tokio::time::sleep; @@ -1086,19 +1086,16 @@ impl ByteStreamServer { .map(|_| ()) .err_tip(|| "Failed to store decompressed data") }; - let decode_fut = async move { - spawn_blocking!("bytestream_decode_compressed_upload", move || { - crate::wire_compression::stream_decode_compressed_upload( - compressed_rx, - wire_compressor, - digest, - digest_function, - decompressed_tx, - ) - }) - .await - .map_err(|e| make_err!(Code::Internal, "Decompression task failed: {}", e))? - }; + // Plain async future: decode progresses at the pace of the client + // upload and the store write without occupying a blocking-pool thread + // for the stream's lifetime. + let decode_fut = crate::wire_compression::stream_decode_compressed_upload( + compressed_rx, + wire_compressor, + digest, + digest_function, + decompressed_tx, + ); let client_stream_fut = process_compressed_client_stream(stream, compressed_tx, &bytes_received); let (client_stream_result, decode_result, store_update_result) = @@ -1207,17 +1204,16 @@ impl ByteStreamServer { .await .err_tip(|| "Failed to read blob for wire compression") }); - let encode_fut = Box::pin(async move { - spawn_blocking!("bytestream_encode_compressed_download", move || { - crate::wire_compression::stream_encode_compressed_download( - raw_rx, - wire_compressor, - compressed_tx, - ) - }) - .await - .map_err(|e| make_err!(Code::Internal, "Compression task failed: {}", e))? - }); + // The encode runs as a plain async future: it must not occupy a + // blocking-pool thread for the stream's lifetime, because it only + // progresses at the client's drain rate. Dropping the returned + // stream drops this future, which tears the encode down exactly + // like the previous task-abort-on-drop did. + let encode_fut = Box::pin(crate::wire_compression::stream_encode_compressed_download( + raw_rx, + wire_compressor, + compressed_tx, + )); let state = Some(ReaderState { max_bytes_per_stream: instance.max_bytes_per_stream, diff --git a/nativelink-service/src/wire_compression.rs b/nativelink-service/src/wire_compression.rs index 1e6c5facd..dc4de4a88 100644 --- a/nativelink-service/src/wire_compression.rs +++ b/nativelink-service/src/wire_compression.rs @@ -78,55 +78,6 @@ impl RemoteCacheCompressionInstances { } } -#[derive(Debug)] -pub struct BufChannelReader { - rx: DropCloserReadHalf, - chunk: Bytes, - chunk_offset: usize, -} - -impl BufChannelReader { - pub const fn new(rx: DropCloserReadHalf) -> Self { - Self { - rx, - chunk: Bytes::new(), - chunk_offset: 0, - } - } - - fn refill_chunk(&mut self) -> std::io::Result { - while self.chunk_offset == self.chunk.len() { - self.chunk = self.rx.blocking_recv().map_err(Error::to_std_err)?; - self.chunk_offset = 0; - if self.chunk.is_empty() { - return Ok(false); - } - } - - Ok(true) - } -} - -impl Read for BufChannelReader { - fn read(&mut self, output: &mut [u8]) -> std::io::Result { - if output.is_empty() { - return Ok(0); - } - - if !self.refill_chunk()? { - return Ok(0); - } - - let chunk_remaining = &self.chunk[self.chunk_offset..]; - let bytes_to_copy = output.len().min(chunk_remaining.len()); - - output[..bytes_to_copy].copy_from_slice(&chunk_remaining[..bytes_to_copy]); - self.chunk_offset += bytes_to_copy; - - Ok(bytes_to_copy) - } -} - /// Resolve a wire compressor from a URI compressor string (as it appears in /// the `compressed-blobs/{compressor}/...` resource name) and validate it /// against whether the instance supports remote cache compression. @@ -291,13 +242,30 @@ pub async fn decompress_batch_update( } } -pub fn stream_decode_compressed_upload( - compressed_rx: DropCloserReadHalf, +/// Decode a client's zstd wire stream into raw bytes on `tx`, asynchronously. +/// +/// Like [`stream_encode_compressed_download`], this must not occupy a tokio +/// blocking-pool thread for the stream's lifetime: the input arrives at the +/// client's upload pace and `tx` drains at the store's write pace, so a +/// blocking implementation parks a pool thread on whichever side is slower +/// for as long as the upload lasts. The zstd frame is consumed incrementally +/// with the raw streaming API instead; per-chunk decode cost is bounded by +/// the channel chunk size, so it runs inline on the async runtime with +/// channel-native backpressure on both sides. +/// +/// Validation semantics match the REAPI compressed-blobs contract: the +/// decoded byte count may never exceed the digest size (checked per chunk so +/// a decompression bomb is rejected as soon as it overshoots), the final +/// count must equal it exactly, and the decoded bytes must hash to `digest`. +pub async fn stream_decode_compressed_upload( + mut compressed_rx: DropCloserReadHalf, wire_compressor: compressor::Value, digest: DigestInfo, digest_function: DigestHasherFunc, mut tx: DropCloserWriteHalf, ) -> Result<(), Error> { + use zstd::stream::raw::{Decoder, InBuffer, Operation, OutBuffer}; + if wire_compressor != compressor::Value::Zstd { return Err(make_input_err!( "Streaming upload decompression only supports zstd, got {:?}", @@ -308,36 +276,62 @@ pub fn stream_decode_compressed_upload( let expected_size = digest.size_bytes(); let mut hasher = digest_function.hasher(); let mut decoded_size = 0u64; - let mut buffer = vec![0u8; zstd::zstd_safe::DCtx::out_size()]; - - let reader = BufChannelReader::new(compressed_rx); - let mut decoder = zstd::stream::read::Decoder::new(reader) + let mut decoder = Decoder::new() .map_err(|e| make_err!(Code::InvalidArgument, "Zstd decompression failed: {}", e))?; + // `DCtx::out_size()` guarantees a full decompressed block always fits, so + // the decoder never stalls for lack of output space within one `run`. + let mut out_buf = vec![0u8; zstd::zstd_safe::DCtx::out_size()]; + // Last input-size hint from the decoder: nonzero at input EOF means the + // stream ended in the middle of a frame and must be rejected. + let mut frame_input_hint = 0usize; loop { - let read = decoder - .read(&mut buffer) - .map_err(|e| make_err!(Code::InvalidArgument, "Zstd decompression failed: {}", e))?; - if read == 0 { - break; + let chunk = compressed_rx + .recv() + .await + .err_tip(|| "Failed to receive compressed data in stream_decode_compressed_upload")?; + if chunk.is_empty() { + break; // EOF. } - let read_u64 = - u64::try_from(read).err_tip(|| "Decoded chunk size was not convertible to u64")?; - decoded_size = decoded_size.checked_add(read_u64).ok_or_else(|| { - make_err!( - Code::InvalidArgument, - "Decoded compressed upload size overflow" - ) - })?; - if decoded_size > expected_size { - return Err(make_err!( - Code::InvalidArgument, - "Decoded compressed upload size {} bytes exceeds digest size {} bytes", - decoded_size, - expected_size - )); + let mut in_buffer = InBuffer::around(&chunk); + loop { + let mut out_buffer = OutBuffer::around(out_buf.as_mut_slice()); + frame_input_hint = decoder.run(&mut in_buffer, &mut out_buffer).map_err(|e| { + make_err!(Code::InvalidArgument, "Zstd decompression failed: {}", e) + })?; + let produced = Bytes::copy_from_slice(out_buffer.as_slice()); + // A completely full output buffer means the decoder may still + // have buffered output to flush, even with no input left. + let output_was_full = produced.len() == out_buf.len(); + if !produced.is_empty() { + let produced_u64 = u64::try_from(produced.len()) + .err_tip(|| "Decoded chunk size was not convertible to u64")?; + decoded_size = decoded_size.checked_add(produced_u64).ok_or_else(|| { + make_err!( + Code::InvalidArgument, + "Decoded compressed upload size overflow" + ) + })?; + if decoded_size > expected_size { + return Err(make_err!( + Code::InvalidArgument, + "Decoded compressed upload size {} bytes exceeds digest size {} bytes", + decoded_size, + expected_size + )); + } + hasher.update(&produced); + tx.send(produced).await?; + } + if in_buffer.pos() == in_buffer.src.len() && !output_was_full { + break; + } } - hasher.update(&buffer[..read]); - tx.blocking_send(Bytes::copy_from_slice(&buffer[..read]))?; + } + if frame_input_hint != 0 { + return Err(make_err!( + Code::InvalidArgument, + "Compressed upload stream ended in the middle of a zstd frame" + )); } if decoded_size != expected_size { @@ -363,11 +357,31 @@ pub fn stream_decode_compressed_upload( Ok(()) } -pub fn stream_encode_compressed_download( - raw_rx: DropCloserReadHalf, +/// Encode a raw byte stream into a single zstd frame on `tx`, asynchronously. +/// +/// This runs entirely on the async runtime and must never occupy a tokio +/// blocking-pool thread for the stream's lifetime: `tx` drains at the gRPC +/// client's pace, so a blocking implementation (blocking reads from `raw_rx` +/// plus `blocking_send` into `tx`) parks one pool thread per concurrent +/// compressed download until the client finishes. Enough concurrent downloads +/// with slow consumers then exhaust the blocking pool and starve every other +/// `spawn_blocking` user (filesystem store I/O, upload decode, credential +/// resolution). The zstd frame is instead produced incrementally with the raw +/// streaming API: the CPU cost per iteration is bounded by the channel chunk +/// size (small — micro/milliseconds), so it is acceptable inline on a worker +/// thread, and `tx.send(...).await` gives backpressure without a parked +/// thread. +/// +/// The output is one well-formed zstd frame, identical in wire format to what +/// `zstd::stream::read::Encoder` produces (both drive `ZSTD_compressStream` +/// on a fresh `CCtx`). +pub async fn stream_encode_compressed_download( + mut raw_rx: DropCloserReadHalf, wire_compressor: compressor::Value, mut tx: DropCloserWriteHalf, ) -> Result<(), Error> { + use zstd::stream::raw::{Encoder, InBuffer, Operation, OutBuffer}; + if wire_compressor != compressor::Value::Zstd { return Err(make_input_err!( "Streaming download compression only supports zstd, got {:?}", @@ -375,18 +389,47 @@ pub fn stream_encode_compressed_download( )); } - let reader = BufChannelReader::new(raw_rx); - let mut encoder = zstd::stream::read::Encoder::new(reader, ZSTD_COMPRESSION_LEVEL) + let mut encoder = Encoder::new(ZSTD_COMPRESSION_LEVEL) .map_err(|e| make_err!(Code::Internal, "Zstd compression failed: {}", e))?; - let mut buffer = vec![0u8; 64 * 1024]; + // `CCtx::out_size()` guarantees a full compressed block always fits, so + // the encoder never stalls for lack of output space within one `run`. + let mut out_buf = vec![0u8; zstd::zstd_safe::CCtx::out_size()]; loop { - let read = encoder - .read(&mut buffer) + let chunk = raw_rx + .recv() + .await + .err_tip(|| "Failed to receive raw data in stream_encode_compressed_download")?; + if chunk.is_empty() { + break; // EOF. + } + let mut in_buffer = InBuffer::around(&chunk); + while in_buffer.pos() < in_buffer.src.len() { + let mut out_buffer = OutBuffer::around(out_buf.as_mut_slice()); + encoder + .run(&mut in_buffer, &mut out_buffer) + .map_err(|e| make_err!(Code::Internal, "Zstd compression failed: {}", e))?; + let produced = out_buffer.as_slice(); + if !produced.is_empty() { + tx.send(Bytes::copy_from_slice(produced)).await?; + } + } + } + + // Finish the frame: flush any internally buffered compressed data plus + // the frame epilogue. `finish` reports the bytes still pending, so loop + // until it reports none. + loop { + let mut out_buffer = OutBuffer::around(out_buf.as_mut_slice()); + let remaining = encoder + .finish(&mut out_buffer, true) .map_err(|e| make_err!(Code::Internal, "Zstd compression failed: {}", e))?; - if read == 0 { + let produced = out_buffer.as_slice(); + if !produced.is_empty() { + tx.send(Bytes::copy_from_slice(produced)).await?; + } + if remaining == 0 { break; } - tx.blocking_send(Bytes::copy_from_slice(&buffer[..read]))?; } tx.send_eof() diff --git a/nativelink-service/tests/wire_compression_test.rs b/nativelink-service/tests/wire_compression_test.rs index f829c7dfa..592f0c565 100644 --- a/nativelink-service/tests/wire_compression_test.rs +++ b/nativelink-service/tests/wire_compression_test.rs @@ -15,20 +15,20 @@ //! Unit tests for the REAPI wire-compression helpers shared by the //! `ByteStream` and CAS services: compressor URI resolution, zstd //! encode/decode with size validation, the zero-copy identity batch-update -//! path, and the blocking [`BufChannelReader`] adapter used by the streaming -//! zstd encoder/decoder. +//! path, and blocking-pool isolation of the async streaming zstd encoder. -use std::io::Read; +use core::time::Duration; use bytes::Bytes; use nativelink_error::{Code, Error}; use nativelink_macro::nativelink_test; use nativelink_proto::build::bazel::remote::execution::v2::compressor; use nativelink_service::wire_compression::{ - BufChannelReader, compress, decompress, decompress_batch_update, resolve_wire_compressor, + compress, decompress, decompress_batch_update, resolve_wire_compressor, + stream_encode_compressed_download, }; use nativelink_util::buf_channel::make_buf_channel_pair; -use nativelink_util::spawn_blocking; +use nativelink_util::{spawn, spawn_blocking}; use pretty_assertions::assert_eq; #[nativelink_test] @@ -164,38 +164,106 @@ async fn resolve_wire_compressor_accepts_only_advertised_compressors() -> Result Ok(()) } -#[nativelink_test] -async fn buf_channel_reader_reads_partial_and_multiple_chunks() -> Result<(), Error> { - let (mut tx, rx) = make_buf_channel_pair(); - - // BufChannelReader is a blocking `Read` adapter (it uses - // `blocking_recv`), so it must run off the async runtime — exactly how - // the streaming zstd encoder/decoder drive it in production. - let reader_task = spawn_blocking!("buf_channel_reader_test", move || { - let mut reader = BufChannelReader::new(rx); - - let mut first_partial_chunk = [0; 2]; - reader - .read_exact(&mut first_partial_chunk) - .expect("failed to read first partial chunk"); - assert_eq!(&first_partial_chunk, b"ab"); - - let mut remaining_chunks = [0; 6]; - reader - .read_exact(&mut remaining_chunks) - .expect("failed to read remaining chunks"); - assert_eq!(&remaining_chunks, b"cdefgh"); - - let mut eof_probe = [0; 1]; - assert_eq!(reader.read(&mut eof_probe).expect("failed to read EOF"), 0); - }); +/// A compressed-download encode stream lives at the gRPC CLIENT's drain rate: +/// its output channel only empties as fast as the client reads, and a blob can +/// take arbitrarily long to stream out. If each such stream occupies a tokio +/// blocking-pool thread for its whole lifetime, N concurrent downloads with +/// slow consumers occupy N threads and every unrelated `spawn_blocking` user +/// (filesystem store I/O, upload decode, credential providers) queues behind +/// streams that may not finish for minutes. This test wires up encode streams +/// exactly the way `ByteStreamServer::inner_read_compressed` does, with more +/// held streams than blocking-pool threads, and asserts that a trivial +/// unrelated `spawn_blocking` closure still gets to run. +#[test] +#[expect( + clippy::disallowed_methods, + reason = "the defect under test is blocking-pool capacity, so the test needs \ + a dedicated runtime with a deterministically small blocking pool" +)] +fn held_compressed_download_streams_must_not_starve_blocking_pool() { + // More encode streams than blocking-pool threads. The streams never + // complete during the test: their input never reaches EOF and their + // consumer never reads, so any per-stream thread is held indefinitely. + // The blocking pool schedules FIFO, so if each stream occupies a thread + // the sentinel spawned afterwards can never run — no sleeps or timing + // races are needed for the starvation to be deterministic. + const MAX_BLOCKING_THREADS: usize = 4; + const NUM_STREAMS: usize = 8; + const SENTINEL_TIMEOUT: Duration = Duration::from_secs(2); + const OVERALL_TIMEOUT: Duration = Duration::from_secs(30); + + let runtime = tokio::runtime::Builder::new_multi_thread() + .worker_threads(2) + .max_blocking_threads(MAX_BLOCKING_THREADS) + .enable_all() + .build() + .expect("failed to build test runtime"); + + runtime.block_on(async { + tokio::time::timeout(OVERALL_TIMEOUT, async { + // Held alive for the duration of the sentinel probe: dropping the + // feeders, encode handles, or receivers early would tear the + // streams down and release any held threads. + let mut held_streams = Vec::with_capacity(NUM_STREAMS); + for _ in 0..NUM_STREAMS { + let (mut raw_tx, raw_rx) = make_buf_channel_pair(); + let (compressed_tx, compressed_rx) = make_buf_channel_pair(); + + // Feed incompressible pseudo-random data forever (no EOF) so + // the encoder always has input and keeps producing output. + // The output channel has a small fixed capacity and nothing + // reads `compressed_rx`, so the encoder soon waits on a full + // output channel — the "slow client" that holds the stream + // open. Async sends here mean the feeders themselves never + // occupy blocking-pool threads. + let feeder = spawn!("test_raw_data_feeder", async move { + let mut state = 0x9E37_79B9_7F4A_7C15_u64; + loop { + let mut chunk = vec![0u8; 64 * 1024]; + for word in chunk.chunks_exact_mut(8) { + state = state + .wrapping_mul(6_364_136_223_846_793_005) + .wrapping_add(1_442_695_040_888_963_407); + word.copy_from_slice(&state.to_le_bytes()); + } + if raw_tx.send(Bytes::from(chunk)).await.is_err() { + break; + } + } + }); - tx.send(Bytes::from_static(b"abc")).await?; - tx.send(Bytes::from_static(b"defgh")).await?; - tx.send_eof()?; + // Start the encode stream the same way + // `ByteStreamServer::inner_read_compressed` does: as a plain + // async future on the runtime, never on the blocking pool. + let encode_task = spawn!( + "test_encode_compressed_download", + stream_encode_compressed_download( + raw_rx, + compressor::Value::Zstd, + compressed_tx, + ) + ); - reader_task + held_streams.push((feeder, encode_task, compressed_rx)); + } + + // Unrelated blocking work must not starve behind held + // compressed-download streams. + let sentinel = spawn_blocking!("test_blocking_pool_sentinel", || ()); + let sentinel_result = tokio::time::timeout(SENTINEL_TIMEOUT, sentinel).await; + assert!( + sentinel_result.is_ok(), + "unrelated blocking work must not starve behind {NUM_STREAMS} held \ + compressed-download streams on a {MAX_BLOCKING_THREADS}-thread blocking pool" + ); + + drop(held_streams); + }) .await - .map_err(|e| Error::new(Code::Internal, format!("reader task panicked: {e:?}")))?; - Ok(()) + .expect("test exceeded overall timeout"); + }); + + // Encode streams stuck on a full output channel park their blocking-pool + // threads indefinitely; a normal runtime drop would join them and hang. + runtime.shutdown_background(); } From 7d3e58f27f8800fc536db3141596106339ef683a Mon Sep 17 00:00:00 2001 From: Aman Kumar Date: Fri, 17 Jul 2026 16:46:20 +0100 Subject: [PATCH 17/84] Release NativeLink v1.6.2 (#2575) --- CHANGELOG.md | 42 +++++++++++++++++++ Cargo.lock | 26 ++++++------ Cargo.toml | 2 +- MODULE.bazel | 2 +- nativelink-config/Cargo.toml | 2 +- nativelink-error/Cargo.toml | 2 +- nativelink-macro/Cargo.toml | 2 +- nativelink-metric/Cargo.toml | 2 +- .../nativelink-metric-macro-derive/Cargo.toml | 2 +- nativelink-proto/Cargo.toml | 2 +- nativelink-redis-tester/Cargo.toml | 2 +- nativelink-scheduler/Cargo.toml | 2 +- nativelink-service/Cargo.toml | 2 +- nativelink-store/Cargo.toml | 2 +- nativelink-test/fuzz/Cargo.lock | 10 ++--- nativelink-util/Cargo.toml | 2 +- nativelink-worker/Cargo.toml | 2 +- 17 files changed, 74 insertions(+), 32 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0588c1eda..fab1dedec 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,48 @@ All notable changes to this project will be documented in this file. +## [1.6.2](https://github.com/TraceMachina/nativelink/compare/v1.6.1..v1.6.2) - 2026-07-17 + +### ⛰️ Features + +- Add fallback match interval to simple scheduler ([#2557](https://github.com/TraceMachina/nativelink/issues/2557)) - ([7a4279b](https://github.com/TraceMachina/nativelink/commit/7a4279b4e51cfb6064937b50f763e3c13733db0a)) +- Add opt-in small-blob read coalescing to GrpcStore ([#2540](https://github.com/TraceMachina/nativelink/issues/2540)) - ([4be35e6](https://github.com/TraceMachina/nativelink/commit/4be35e6dccb7fbc08d414ca029c592e9e7853cbf)) + +### 🐛 Bug Fixes + +- Remove quoting around NetApp ([#2556](https://github.com/TraceMachina/nativelink/issues/2556)) - ([65e2ec2](https://github.com/TraceMachina/nativelink/commit/65e2ec2004ca28e3d9273959c7a809f9b0dcb0e0)) +- Fix skopeo login issues on image publish ([#2549](https://github.com/TraceMachina/nativelink/issues/2549)) - ([e72490c](https://github.com/TraceMachina/nativelink/commit/e72490cc3e388a944f177ce47bca46ecbb027180)) +- Handle zero digests in CompressionStore ([#2548](https://github.com/TraceMachina/nativelink/issues/2548)) - ([b08995d](https://github.com/TraceMachina/nativelink/commit/b08995de0447d92fc5a47f71af1d83d4c4da142c)) + +### 📚 Documentation + +- *(config-reference)* regenerate for NativeLink v1.6.0 ([#2553](https://github.com/TraceMachina/nativelink/issues/2553)) - ([f871377](https://github.com/TraceMachina/nativelink/commit/f871377c09ed6fea7538d372e3a3cd2240c231e2)) +- Adds weekly and "everything" docs regen ([#2551](https://github.com/TraceMachina/nativelink/issues/2551)) - ([2a061ba](https://github.com/TraceMachina/nativelink/commit/2a061ba865d2509decc762f11fbdbcf2871d04f9)) +- Remove StoreSpec docs ([#2545](https://github.com/TraceMachina/nativelink/issues/2545)) - ([fb37e86](https://github.com/TraceMachina/nativelink/commit/fb37e86317de19e757ff4e276f2021f9ae0b79cc)) +- Add opt-in subtree-keyed caching to the worker DirectoryCache ([#2541](https://github.com/TraceMachina/nativelink/issues/2541)) - ([9f9451e](https://github.com/TraceMachina/nativelink/commit/9f9451e02cfc1adf0c7830fa4003ab328f894d56)) + +### 🧪 Testing & CI + +- Fix AsyncFixedBuffer with zstd compression, remove unused BufChannelReader adapter ([#2574](https://github.com/TraceMachina/nativelink/issues/2574)) - ([b01cb0d](https://github.com/TraceMachina/nativelink/commit/b01cb0db985a8c5356b58751460981c97bfffd6f)) +- Reject duplicate store names in StoreManager ([#2533](https://github.com/TraceMachina/nativelink/issues/2533)) - ([2106b1a](https://github.com/TraceMachina/nativelink/commit/2106b1a8481c502e7069b2635f39d83a54b37268)) +- Add free-disk to Redis store tester ([#2543](https://github.com/TraceMachina/nativelink/issues/2543)) - ([4b1de2c](https://github.com/TraceMachina/nativelink/commit/4b1de2cf24a86429480235d4641668531ede4e72)) +- Fix flakey MongoDB test ([#2535](https://github.com/TraceMachina/nativelink/issues/2535)) - ([b8e8171](https://github.com/TraceMachina/nativelink/commit/b8e81711a8a5d3048884ef94667415d4afe52a2a)) + +### ⚙️ Miscellaneous + +- Remove more unused items ([#2561](https://github.com/TraceMachina/nativelink/issues/2561)) - ([8a8be18](https://github.com/TraceMachina/nativelink/commit/8a8be1879d70eceff4caf1b73d6ae63c61fddc5d)) +- Don't update config reference without changes ([#2562](https://github.com/TraceMachina/nativelink/issues/2562)) - ([48d9b96](https://github.com/TraceMachina/nativelink/commit/48d9b961e5403a904708e3394eead2a26df30d81)) +- Tag config regen builds to our bot account ([#2554](https://github.com/TraceMachina/nativelink/issues/2554)) - ([5003676](https://github.com/TraceMachina/nativelink/commit/50036764f6e87fe0518f62eb2ee6b2924f7fadd7)) +- Enforce Vale settings for config as well as markdown ([#2552](https://github.com/TraceMachina/nativelink/issues/2552)) - ([27fb8ad](https://github.com/TraceMachina/nativelink/commit/27fb8adde6554de68d452ef87535822ad4330680)) +- curl 8.5.0-2ubuntu10.11 security update ([#2544](https://github.com/TraceMachina/nativelink/issues/2544)) - ([a8e383a](https://github.com/TraceMachina/nativelink/commit/a8e383ae5e8bd78c8b50fdbad17e7376428a3b3e)) + +### ⬆️ Bumps & Version Updates + +- Upgrade anyhow to 1.0.103 for RUSTSEC-2026-0190 ([#2569](https://github.com/TraceMachina/nativelink/issues/2569)) - ([6675acb](https://github.com/TraceMachina/nativelink/commit/6675acb9ed130ee11f174772a0677500944c8049)) +- Upgrade quick-xml to 0.41 for RUSTSEC-2026-0194 ([#2566](https://github.com/TraceMachina/nativelink/issues/2566)) - ([b173e4b](https://github.com/TraceMachina/nativelink/commit/b173e4be5a65abf56e2c0d620aa7ea23b9107900)) +- Upgrade serde_with to 3.21 for GHSA-7gcf-g7xr-8hxj ([#2559](https://github.com/TraceMachina/nativelink/issues/2559)) - ([b18311f](https://github.com/TraceMachina/nativelink/commit/b18311f46f341bf30e4067a04b9ed93866b9bd46)) +- Update nixpkgs so we can drop our patched renovate ([#2537](https://github.com/TraceMachina/nativelink/issues/2537)) - ([ad0d83d](https://github.com/TraceMachina/nativelink/commit/ad0d83dd80d4f1c88f40468722cfb950bd7e87af)) + ## [1.6.1](https://github.com/TraceMachina/nativelink/compare/v1.6.0..v1.6.1) - 2026-07-08 ### ⛰️ Features diff --git a/Cargo.lock b/Cargo.lock index f968b6792..2eea85292 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2963,7 +2963,7 @@ checksum = "1d87ecb2933e8aeadb3e3a02b828fed80a7528047e68b4f424523a0981a3a084" [[package]] name = "nativelink" -version = "1.6.1" +version = "1.6.2" dependencies = [ "async-lock", "axum", @@ -2994,7 +2994,7 @@ dependencies = [ [[package]] name = "nativelink-config" -version = "1.6.1" +version = "1.6.2" dependencies = [ "byte-unit", "humantime", @@ -3013,7 +3013,7 @@ dependencies = [ [[package]] name = "nativelink-error" -version = "1.6.1" +version = "1.6.2" dependencies = [ "base64 0.22.1", "mongodb", @@ -3037,7 +3037,7 @@ dependencies = [ [[package]] name = "nativelink-macro" -version = "1.6.1" +version = "1.6.2" dependencies = [ "proc-macro2", "quote", @@ -3046,7 +3046,7 @@ dependencies = [ [[package]] name = "nativelink-metric" -version = "1.6.1" +version = "1.6.2" dependencies = [ "async-lock", "nativelink-metric-macro-derive", @@ -3057,7 +3057,7 @@ dependencies = [ [[package]] name = "nativelink-metric-macro-derive" -version = "1.6.1" +version = "1.6.2" dependencies = [ "proc-macro2", "quote", @@ -3066,7 +3066,7 @@ dependencies = [ [[package]] name = "nativelink-proto" -version = "1.6.1" +version = "1.6.2" dependencies = [ "derive_more 2.1.0", "prost", @@ -3078,7 +3078,7 @@ dependencies = [ [[package]] name = "nativelink-redis-tester" -version = "1.6.1" +version = "1.6.2" dependencies = [ "either", "nativelink-util", @@ -3091,7 +3091,7 @@ dependencies = [ [[package]] name = "nativelink-scheduler" -version = "1.6.1" +version = "1.6.2" dependencies = [ "async-lock", "async-trait", @@ -3128,7 +3128,7 @@ dependencies = [ [[package]] name = "nativelink-service" -version = "1.6.1" +version = "1.6.2" dependencies = [ "async-lock", "async-trait", @@ -3172,7 +3172,7 @@ dependencies = [ [[package]] name = "nativelink-store" -version = "1.6.1" +version = "1.6.2" dependencies = [ "async-lock", "async-trait", @@ -3250,7 +3250,7 @@ dependencies = [ [[package]] name = "nativelink-util" -version = "1.6.1" +version = "1.6.2" dependencies = [ "anyhow", "async-trait", @@ -3312,7 +3312,7 @@ dependencies = [ [[package]] name = "nativelink-worker" -version = "1.6.1" +version = "1.6.2" dependencies = [ "async-lock", "async-trait", diff --git a/Cargo.toml b/Cargo.toml index 4515689cf..6365c9234 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -11,7 +11,7 @@ resolver = "2" edition = "2024" name = "nativelink" rust-version = "1.93.1" -version = "1.6.1" +version = "1.6.2" [profile.release] lto = true diff --git a/MODULE.bazel b/MODULE.bazel index ce93ef403..f17b392ca 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -1,6 +1,6 @@ module( name = "nativelink", - version = "1.6.1", + version = "1.6.2", compatibility_level = 0, ) diff --git a/nativelink-config/Cargo.toml b/nativelink-config/Cargo.toml index bb46a363a..a35cb017f 100644 --- a/nativelink-config/Cargo.toml +++ b/nativelink-config/Cargo.toml @@ -4,7 +4,7 @@ lints.workspace = true [package] edition = "2024" name = "nativelink-config" -version = "1.6.1" +version = "1.6.2" [dependencies] nativelink-error = { path = "../nativelink-error" } diff --git a/nativelink-error/Cargo.toml b/nativelink-error/Cargo.toml index 42be30079..3455c3fa2 100644 --- a/nativelink-error/Cargo.toml +++ b/nativelink-error/Cargo.toml @@ -8,7 +8,7 @@ autoexamples = false autotests = true edition = "2024" name = "nativelink-error" -version = "1.6.1" +version = "1.6.2" [dependencies] nativelink-metric = { path = "../nativelink-metric" } diff --git a/nativelink-macro/Cargo.toml b/nativelink-macro/Cargo.toml index 455a22ca3..26734588f 100644 --- a/nativelink-macro/Cargo.toml +++ b/nativelink-macro/Cargo.toml @@ -4,7 +4,7 @@ lints.workspace = true [package] edition = "2024" name = "nativelink-macro" -version = "1.6.1" +version = "1.6.2" [lib] proc-macro = true diff --git a/nativelink-metric/Cargo.toml b/nativelink-metric/Cargo.toml index 9a922cacb..df71ab454 100644 --- a/nativelink-metric/Cargo.toml +++ b/nativelink-metric/Cargo.toml @@ -4,7 +4,7 @@ lints.workspace = true [package] edition = "2024" name = "nativelink-metric" -version = "1.6.1" +version = "1.6.2" [dependencies] nativelink-metric-macro-derive = { path = "nativelink-metric-macro-derive" } diff --git a/nativelink-metric/nativelink-metric-macro-derive/Cargo.toml b/nativelink-metric/nativelink-metric-macro-derive/Cargo.toml index cabb4b44a..60d9c31c3 100644 --- a/nativelink-metric/nativelink-metric-macro-derive/Cargo.toml +++ b/nativelink-metric/nativelink-metric-macro-derive/Cargo.toml @@ -1,7 +1,7 @@ [package] edition = "2024" name = "nativelink-metric-macro-derive" -version = "1.6.1" +version = "1.6.2" [lib] proc-macro = true diff --git a/nativelink-proto/Cargo.toml b/nativelink-proto/Cargo.toml index bf8131fb3..c389c531b 100644 --- a/nativelink-proto/Cargo.toml +++ b/nativelink-proto/Cargo.toml @@ -2,7 +2,7 @@ [package] edition = "2024" name = "nativelink-proto" -version = "1.6.1" +version = "1.6.2" [lib] doctest = false # because some of the generated protos have things that look like doctests but break diff --git a/nativelink-redis-tester/Cargo.toml b/nativelink-redis-tester/Cargo.toml index f2fcfc665..e17212d56 100644 --- a/nativelink-redis-tester/Cargo.toml +++ b/nativelink-redis-tester/Cargo.toml @@ -4,7 +4,7 @@ lints.workspace = true [package] edition = "2024" name = "nativelink-redis-tester" -version = "1.6.1" +version = "1.6.2" [dependencies] nativelink-util = { path = "../nativelink-util" } diff --git a/nativelink-scheduler/Cargo.toml b/nativelink-scheduler/Cargo.toml index 43d97dad5..34870911a 100644 --- a/nativelink-scheduler/Cargo.toml +++ b/nativelink-scheduler/Cargo.toml @@ -4,7 +4,7 @@ lints.workspace = true [package] edition = "2024" name = "nativelink-scheduler" -version = "1.6.1" +version = "1.6.2" [dependencies] nativelink-config = { path = "../nativelink-config" } diff --git a/nativelink-service/Cargo.toml b/nativelink-service/Cargo.toml index be34c0102..b03e5e4f6 100644 --- a/nativelink-service/Cargo.toml +++ b/nativelink-service/Cargo.toml @@ -4,7 +4,7 @@ lints.workspace = true [package] edition = "2024" name = "nativelink-service" -version = "1.6.1" +version = "1.6.2" [dependencies] nativelink-config = { path = "../nativelink-config" } diff --git a/nativelink-store/Cargo.toml b/nativelink-store/Cargo.toml index 538b4a083..a276d2070 100644 --- a/nativelink-store/Cargo.toml +++ b/nativelink-store/Cargo.toml @@ -4,7 +4,7 @@ lints.workspace = true [package] edition = "2024" name = "nativelink-store" -version = "1.6.1" +version = "1.6.2" [dependencies] nativelink-config = { path = "../nativelink-config" } diff --git a/nativelink-test/fuzz/Cargo.lock b/nativelink-test/fuzz/Cargo.lock index 95d52156d..cca966b96 100644 --- a/nativelink-test/fuzz/Cargo.lock +++ b/nativelink-test/fuzz/Cargo.lock @@ -1038,7 +1038,7 @@ dependencies = [ [[package]] name = "nativelink-config" -version = "1.6.1" +version = "1.6.2" dependencies = [ "byte-unit", "humantime", @@ -1053,7 +1053,7 @@ dependencies = [ [[package]] name = "nativelink-error" -version = "1.6.1" +version = "1.6.2" dependencies = [ "base64", "mongodb", @@ -1085,7 +1085,7 @@ dependencies = [ [[package]] name = "nativelink-metric" -version = "1.6.1" +version = "1.6.2" dependencies = [ "async-lock", "nativelink-metric-macro-derive", @@ -1096,7 +1096,7 @@ dependencies = [ [[package]] name = "nativelink-metric-macro-derive" -version = "1.6.1" +version = "1.6.2" dependencies = [ "proc-macro2", "quote", @@ -1105,7 +1105,7 @@ dependencies = [ [[package]] name = "nativelink-proto" -version = "1.6.1" +version = "1.6.2" dependencies = [ "derive_more", "prost", diff --git a/nativelink-util/Cargo.toml b/nativelink-util/Cargo.toml index 0c75df3be..89fa102d3 100644 --- a/nativelink-util/Cargo.toml +++ b/nativelink-util/Cargo.toml @@ -4,7 +4,7 @@ lints.workspace = true [package] edition = "2024" name = "nativelink-util" -version = "1.6.1" +version = "1.6.2" [dependencies] nativelink-config = { path = "../nativelink-config" } diff --git a/nativelink-worker/Cargo.toml b/nativelink-worker/Cargo.toml index 9263e36c1..2382e3515 100644 --- a/nativelink-worker/Cargo.toml +++ b/nativelink-worker/Cargo.toml @@ -4,7 +4,7 @@ lints.workspace = true [package] edition = "2024" name = "nativelink-worker" -version = "1.6.1" +version = "1.6.2" [features] nix = [] From 9717f2fb432c0b8cdd26b078d4851040be64abdd Mon Sep 17 00:00:00 2001 From: Aman Kumar Date: Fri, 17 Jul 2026 18:25:46 +0100 Subject: [PATCH 18/84] Fix skopeo login on ubuntu-24.04 for all image workflows (#2577) --- .github/actions/test-and-upload-image/action.yaml | 5 +++++ .github/workflows/image.yaml | 7 ------- 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/.github/actions/test-and-upload-image/action.yaml b/.github/actions/test-and-upload-image/action.yaml index c90ede9c9..21bff1073 100644 --- a/.github/actions/test-and-upload-image/action.yaml +++ b/.github/actions/test-and-upload-image/action.yaml @@ -23,6 +23,11 @@ inputs: runs: using: "composite" steps: + - name: Upgrade registries config to v2 + run: | + echo 'unqualified-search-registries = ["docker.io", "quay.io"]' | sudo tee /etc/containers/registries.conf + shell: bash -euo pipefail {0} + # FIXME: merge the multi-arch and single-arch paths - name: Test multi-arch image if: ${{ inputs.multi-arch == 'true' && inputs.testing == 'true' }} diff --git a/.github/workflows/image.yaml b/.github/workflows/image.yaml index 37a773e55..e5f473891 100644 --- a/.github/workflows/image.yaml +++ b/.github/workflows/image.yaml @@ -51,13 +51,6 @@ jobs: uses: >- # v6.0.2 actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd - # Skopeo login otherwise complains with - # "get credentials: loading registries configuration \"/etc/containers/registries.conf\": registries.conf must be in v2 format but is in v1" - - name: Upgrade registries config to v2 - run: | - cat /etc/containers/registries.conf - echo "unqualified-search-registries = [\"docker.io\", \"quay.io\"]" | sudo tee /etc/containers/registries.conf - - name: Prepare Worker uses: ./.github/actions/prepare-nix with: From 9daa989b24330a09f570c623d13331fd293e5062 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Fri, 17 Jul 2026 20:01:47 +0100 Subject: [PATCH 19/84] docs(config-reference): regenerate for NativeLink v1.6.2 (#2576) Co-authored-by: config reference bot --- .../reference/nativelink-config/index.mdx | 52 +- .../reference/nativelink-config/v1.6.1.mdx | 1521 +++++++++++++++++ web/apps/docs/lib/config-versions.ts | 14 +- 3 files changed, 1566 insertions(+), 21 deletions(-) create mode 100644 web/apps/docs/content/docs/reference/nativelink-config/v1.6.1.mdx diff --git a/web/apps/docs/content/docs/reference/nativelink-config/index.mdx b/web/apps/docs/content/docs/reference/nativelink-config/index.mdx index e6c7307c2..beb6bb0a7 100644 --- a/web/apps/docs/content/docs/reference/nativelink-config/index.mdx +++ b/web/apps/docs/content/docs/reference/nativelink-config/index.mdx @@ -5,14 +5,14 @@ full: true --- {/* AUTOGENERATED — do not edit by hand. - Source: nativelink-config @ v1.6.1 (23e960dc) + Source: nativelink-config @ v1.6.2 (9717f2fb) Regenerate from web/: bun --filter @nativelink/docs gen:config-reference */} -This is the canonical NativeLink configuration reference for **v1.6.1**. +This is the canonical NativeLink configuration reference for **v1.6.2**. It is autogenerated from the Rust config crate -([`nativelink-config/src`](https://github.com/TraceMachina/nativelink/tree/v1.6.1/nativelink-config/src)) via the `build-schema` binary, so +([`nativelink-config/src`](https://github.com/TraceMachina/nativelink/tree/v1.6.2/nativelink-config/src)) via the `build-schema` binary, so it can never drift from what the binary actually deserializes. ## Top-level fields @@ -451,7 +451,7 @@ On uploads it will mirror data to both `fast` and `slow` stores. WARNING: If you need data to always exist in the `slow` store for something like remote execution, be careful because this store will never check to see if the objects exist in the -`slow` store if it exists in the `fast` store (i.e., it assumes +`slow` store if it exists in the `fast` store (i.e. it assumes that if an object exists in the `fast` store it will exist in the `slow` store). @@ -552,7 +552,7 @@ in one store and large objects in another store. This should only be used if the size field is the real size of the content, in other words, don't use on AC (Action Cache) stores. Any store where you can safely use `VerifySpec.verify_size = true`, this store should be safe -to use (i.e., CAS stores). +to use (i.e. CAS stores). **Example JSON5 config:** ```json5 @@ -576,7 +576,7 @@ to use (i.e., CAS stores). ### `grpc` -This store will pass-through calls to another GRPC store. This store +This store will pass-through calls to another gRPC store. This store is not designed to be used as a sub-store of another store, but it does satisfy the interface and will likely work. @@ -817,8 +817,8 @@ See [`experimental_cloud_object_store`](#experimental_cloud_object_store-1) for | Field | Type | Required | Default | Description | | --- | --- | --- | --- | --- | -| `content_path` | string | Yes | — | Path on the system where to store the actual content. This is where the bulk of the data will be placed. On service startup this folder will be scanned and all files will be added to the cache. In the event one of the files doesn't match the criteria, the file will be deleted. | -| `temp_path` | string | Yes | — | A temporary location of where files that are being uploaded or deleted will be placed while the content cannot be guaranteed to be accurate. This location must be on the same block device as `content_path` so atomic moves can happen (i.e., move without copy). All files in this folder will be deleted on every startup. | +| `content_path` | string | Yes | — | Path on the system where to store the actual content. This is where the bulk of the data will be placed. On service boot this folder will be scanned and all files will be added to the cache. In the event one of the files doesn't match the criteria, the file will be deleted. | +| `temp_path` | string | Yes | — | A temporary location of where files that are being uploaded or deleted will be placed while the content cannot be guaranteed to be accurate. This location must be on the same block device as `content_path` so atomic moves can happen (i.e. move without copy). All files in this folder will be deleted on every startup. | | `read_buffer_size` | integer (uint32) | — | 32k | Buffer size to use when reading files. Generally this should be left to the default value except for testing. | | `eviction_policy` | [EvictionPolicy](#evictionpolicy) | — | — | Policy used to evict items out of the store. Failure to set this value will cause items to never be removed from the store causing infinite memory usage. | | `block_size` | integer (uint64) | — | 4kb | The block size of the filesystem for the running machine value is used to determine an entry's actual size on disk consumed For a 4KB block size filesystem, a 1B file actually consumes 4KB | @@ -842,16 +842,17 @@ See [`experimental_cloud_object_store`](#experimental_cloud_object_store-1) for | Field | Type | Required | Default | Description | | --- | --- | --- | --- | --- | -| `instance_name` | string | — | `""` | Instance name for GRPC calls. Proxy calls will have the `instance_name` changed to this. | +| `instance_name` | string | — | `""` | Instance name for gRPC calls. Proxy calls will have the `instance_name` changed to this. | | `endpoints` | array of [GrpcEndpoint](#grpcendpoint) | Yes | — | The endpoint of the grpc connection. | | `store_type` | [StoreType](#storetype) | Yes | — | The type of the upstream store, this ensures that the correct server calls are made. | | `retry` | [Retry](#retry) | — | — | Retry configuration to use when a network request fails. | | `max_concurrent_requests` | integer (uint) | — | `0` | Limit the number of simultaneous upstream requests to this many. A value of zero is treated as unlimited. If the limit is reached the request is queued. | -| `connections_per_endpoint` | integer (uint) | — | `0` | The number of connections to make to each specified endpoint to balance the load over multiple TCP connections. Default 1. | +| `connections_per_endpoint` | integer (uint) | — | 1 | The number of connections to make to each specified endpoint to balance the load over multiple TCP connections. | | `rpc_timeout_s` | integer (uint64) | — | 0 (disabled) | Maximum time (seconds) allowed for a single RPC request (e.g. a `ByteStream.Write` call) before it is cancelled. | | `use_legacy_resource_names` | boolean | — | false | Use legacy `ByteStream` resource name format, omitting the digest function component from the path. | | `headers` | map of string to string | — | — | Static headers to attach to every outgoing gRPC request sent to this store's upstream endpoints. Useful for fixed authentication tokens (e.g. `{"authorization": "Bearer "}`) and other static metadata. | | `forward_headers` | array of string | — | — | Header names to forward from the incoming client request to every outgoing upstream request. The header value is taken from the client request that triggered this store operation. Use this to pass through dynamic credentials such as JWT tokens sent by build clients. | +| `experimental_read_batching` | [GrpcReadBatchingConfig](#grpcreadbatchingconfig) | — | unset (disabled) | Optional and experimental: coalesce small-blob reads into `BatchReadBlobs` RPCs instead of issuing one `ByteStream` `Read` stream per blob. Each `ByteStream` read carries a fixed per-RPC cost, so batching many small reads into a single `BatchReadBlobs` request can dramatically reduce read latency for small blobs. | ## RedisSpec @@ -869,7 +870,7 @@ See [`experimental_cloud_object_store`](#experimental_cloud_object_store-1) for | `health_check_timeout_ms` | integer (uint64) | — | 4000 (4 seconds) | Per-call ceiling for the `check_health` PING in milliseconds. | | `read_chunk_size` | integer (uint) | — | 64KiB | The amount of data to read from the Redis server at a time. This is used to limit the amount of memory used when reading large objects from the Redis server as well as limiting the amount of time a single read operation can take. | | `connection_pool_size` | integer (uint) | — | 3 | The number of connections to keep open to the Redis servers. | -| `max_chunk_uploads_per_update` | integer (uint) | — | 10 | The maximum number of upload chunks to allow per update. This is used to limit the amount of memory used when uploading large objects to the Redis server. A good rule of thumb is to think of the data as: `AVAIL_MEMORY / (read_chunk_size * max_chunk_uploads_per_update) = THORETICAL_MAX_CONCURRENT_UPLOADS` (note: it is a good idea to divide `AVAIL_MAX_MEMORY` by ~10 to account for other memory usage) | +| `max_chunk_uploads_per_update` | integer (uint) | — | 10 | The maximum number of upload chunks to allow per update. This is used to limit the amount of memory used when uploading large objects to the Redis server. A good rule of thumb is to think of the data as: `AVAIL_MEMORY / (read_chunk_size * max_chunk_uploads_per_update) = THORETICAL_MAX_CONCURRENT_UPLOADS` (note: it's a good idea to divide `AVAIL_MAX_MEMORY` by ~10 to account for other memory usage) | | `scan_count` | integer (uint) | — | 10000 | The COUNT value passed when scanning keys in Redis. This is used to hint the amount of work that should be done per response. | | `retry` | [Retry](#retry) | — | — | Retry configuration to use when a network request fails. | | `max_client_permits` | integer (uint) | — | 500 | Maximum number of permitted actions to the Redis store at any one time This stops problems with timeouts due to many, many inflight actions | @@ -904,7 +905,7 @@ Configuration for `ExperimentalMongoDB` store. | Field | Type | Required | Default | Description | | --- | --- | --- | --- | --- | -| `name` | string | — | {Index position in the workers list} | Name of the worker. This is give a more friendly name to a worker for logging and metric publishing. This is also the prefix of the worker id (i.e., "{name}{uuidv6}"). | +| `name` | string | — | {Index position in the workers list} | Name of the worker. This is give a more friendly name to a worker for logging and metric publishing. This is also the prefix of the worker id (i.e. "{name}{uuidv6}"). | | `worker_api_endpoint` | [EndpointConfig](#endpointconfig) | Yes | — | Endpoint which the worker will connect to the scheduler's `WorkerApiService`. | | `max_action_timeout_s` | integer (uint) | — | 20 minutes | The maximum time an action is allowed to run. If a task requests for a timeout longer than this time limit, the task will be rejected. Value in seconds. | | `max_upload_timeout_s` | integer (uint) | — | 10 minutes | Maximum time allowed for uploading action results to CAS after execution completes. If upload takes longer than this, the action fails with `DeadlineExceeded` and may be retried by the scheduler. Value in seconds. | @@ -936,6 +937,7 @@ Configuration for `ExperimentalMongoDB` store. | `allocation_strategy` | [WorkerAllocationStrategy](#workerallocationstrategy) | — | `"least_recently_used"` | The strategy used to assign workers jobs. | | `experimental_backend` | [ExperimentalSimpleSchedulerBackend](#experimentalsimpleschedulerbackend) | — | memory | The storage backend to use for the scheduler. | | `worker_match_logging_interval_s` | integer (int64) | — | `10` | Every N seconds, do logging of worker matching e.g. "worker busy", "can't find any worker" Defaults to 10s. Can be set to `-1` to disable | +| `fallback_match_interval_s` | integer (int64) | — | `5` | Every N seconds, run a worker matching pass even if no task or worker change notification arrived. This is a safety net for missed notifications and for scheduler backends with eventually consistent searches (for example Redis), where an operation that was re-queued may not be visible to the search triggered by its own notification. Without this, such an operation can stay queued until an unrelated event triggers another matching pass. Defaults to 5s. Zero or any negative value disables it. | ## SchedulerGrpcSpec @@ -991,7 +993,7 @@ Listener for HTTP/HTTPS/HTTP2 sockets. | --- | --- | --- | --- | --- | | `cas` | array of [CasServiceConfig](#casserviceconfig) | — | — | The Content Addressable Storage (CAS) backend config. The key is the `instance_name` used in the protocol and the value is the underlying CAS store config. | | `ac` | array of [ActionCacheServiceConfig](#actioncacheserviceconfig) | — | — | The Action Cache (AC) backend config. The key is the `instance_name` used in the protocol and the value is the underlying AC store config. | -| `capabilities` | array of [CapabilitiesServiceConfig](#capabilitiesserviceconfig) | — | — | Capabilities service is required in order to use most of the bazel protocol. This service is used to provide the supported features and versions of this bazel GRPC service. | +| `capabilities` | array of [CapabilitiesServiceConfig](#capabilitiesserviceconfig) | — | — | Capabilities service is required in order to use most of the bazel protocol. This service is used to provide the supported features and versions of this bazel gRPC service. | | `execution` | array of [ExecutionServiceConfig](#executionserviceconfig) | — | — | The remote execution service configuration. NOTE: This service is under development and is currently just a place holder. | | `bytestream` | array of [ByteStreamServiceConfig](#bytestreamserviceconfig) | — | — | This is the service used to stream data to and from the CAS. Bazel's protocol strongly encourages users to use this streaming interface to interact with the CAS when the data is large. | | `fetch` | array of [FetchServiceConfig](#fetchserviceconfig) | — | — | These two are collectively the Remote Asset protocol, but it's defined as two separate services | @@ -1066,7 +1068,7 @@ The total delay is additive, so this example produces 9.525 to 15.875 s of total | `max_retries` | integer (uint) | — | `0` | Maximum number of retries until retrying stops. Setting this to zero will always attempt 1 time, but not retry. | | `delay` | number (float) | — | `0` | Delay in seconds for exponential back off. | | `jitter` | number (float) | — | `0` | Amount of jitter to add as a percentage in decimal form. This will change the formula like: | -| `retry_on_errors` | array of [ErrorCode](#errorcode) | — | — | A list of error codes to retry on, if this is not set then the default error codes to retry on are used. These default codes are the most likely to be non-permanent: `Unknown`; `Cancelled`; `DeadlineExceeded`; `ResourceExhausted`; `Aborted`; `Internal`; `Unavailable`; `DataLoss` | +| `retry_on_errors` | array of [ErrorCode](#errorcode) | — | — | A list of error codes to retry on, if this isn't set then the default error codes to retry on are used. These default codes are the most likely to be non-permanent: `Unknown`; `Cancelled`; `DeadlineExceeded`; `ResourceExhausted`; `Aborted`; `Internal`; `Unavailable`; `DataLoss` | ## ExperimentalOntapS3Spec @@ -1122,7 +1124,7 @@ Configuration for an individual shard of the store. | Field | Type | Required | Default | Description | | --- | --- | --- | --- | --- | -| `address` | string | Yes | — | The endpoint address (i.e. grpc://example.com:443 or grpcs://example.com:443). | +| `address` | string | Yes | — | The endpoint address (i.e. `grpc(s)://example.com:443`). | | `tls_config` | [ClientTlsConfig](#clienttlsconfig) | — | — | The TLS configuration to use to connect to the endpoint (if grpcs). | | `concurrency_limit` | integer (uint) | — | — | The maximum concurrency to allow on this endpoint. | | `connect_timeout_s` | integer (uint64) | — | 30 seconds | Timeout for establishing a TCP connection to the endpoint (seconds). | @@ -1137,6 +1139,18 @@ Configuration for an individual shard of the store. | `"cas"` | The store is content addressable storage. | | `"ac"` | The store is an action cache. | +## GrpcReadBatchingConfig + +Configuration for experimental small-blob read coalescing in a gRPC +store. See [`GrpcSpec::experimental_read_batching`]. + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `max_blob_size_bytes` | integer (uint64) | — | 131072 (128 KiB) | Only blobs at or below this size (in bytes) are eligible for batching. Larger blobs always use the `ByteStream` `Read` path. | +| `max_batch_bytes` | integer (uint64) | — | 3145728 (3 MiB) | Maximum total payload bytes packed into a single `BatchReadBlobs` request. This should leave headroom under the 4 MiB default gRPC message limit for protobuf framing overhead. | +| `dispatch_slots` | integer (uint) | — | 4 | Maximum number of concurrent `BatchReadBlobs` RPCs dispatched by the coalescer. Must be greater than zero. | +| `max_queued_bytes` | integer (uint64) | — | 33554432 (32 MiB) | Bound on the number of payload bytes waiting in the coalescer queue. When exceeded, new read requests bypass batching and fall back to the regular `ByteStream` `Read` path instead of blocking. | + ## RedisMode | Value | Description | @@ -1203,6 +1217,8 @@ One of: | `max_entries` | integer (uint) | — | 1000 | Maximum number of cached directories. | | `max_size_bytes` | integer (uint64) | — | 10737418240 (10 GB) | Maximum total size in bytes for all cached directories (0 = unlimited). | | `cache_root` | string | — | `{work_directory}/../directory_cache` | Base directory for cache storage. This directory will be managed by the worker and should be on the same filesystem as `work_directory`. | +| `experimental_subtree_caching` | boolean | — | false (only root directories are cached; existing behavior) | Optional and experimental: additionally cache every subdirectory by its own `Directory` digest, not just the root directory. REAPI Merkle nodes are content-addressed, so a subtree that is byte-identical between two different roots has the same digest and can be materialized with a single hardlink pass instead of being rebuilt from the CAS. This makes the common "one input file changed out of thousands" case reuse every unchanged subtree. | +| `max_concurrent_fetches` | integer (uint) | — | 64 | Maximum number of concurrent slow-store fetches across ALL directory constructions of this cache. This bound protects backing stores from RPC storms: per-level construction concurrency compounds multiplicatively across tree levels and concurrent actions. | ## PropertyType @@ -1516,6 +1532,6 @@ specified. If anything here disagrees with the binary, the source wins: -- [`stores.rs`](https://github.com/TraceMachina/nativelink/tree/v1.6.1/nativelink-config/src/stores.rs) -- [`cas_server.rs`](https://github.com/TraceMachina/nativelink/tree/v1.6.1/nativelink-config/src/cas_server.rs) -- [`schedulers.rs`](https://github.com/TraceMachina/nativelink/tree/v1.6.1/nativelink-config/src/schedulers.rs) +- [`stores.rs`](https://github.com/TraceMachina/nativelink/tree/v1.6.2/nativelink-config/src/stores.rs) +- [`cas_server.rs`](https://github.com/TraceMachina/nativelink/tree/v1.6.2/nativelink-config/src/cas_server.rs) +- [`schedulers.rs`](https://github.com/TraceMachina/nativelink/tree/v1.6.2/nativelink-config/src/schedulers.rs) diff --git a/web/apps/docs/content/docs/reference/nativelink-config/v1.6.1.mdx b/web/apps/docs/content/docs/reference/nativelink-config/v1.6.1.mdx new file mode 100644 index 000000000..e6c7307c2 --- /dev/null +++ b/web/apps/docs/content/docs/reference/nativelink-config/v1.6.1.mdx @@ -0,0 +1,1521 @@ +--- +title: Configuration reference +description: Every knob in the NativeLink JSON5 configuration — types, defaults, and links to source, autogenerated from the Rust config crate. +full: true +--- + +{/* AUTOGENERATED — do not edit by hand. + Source: nativelink-config @ v1.6.1 (23e960dc) + Regenerate from web/: bun --filter @nativelink/docs gen:config-reference */} + + + +This is the canonical NativeLink configuration reference for **v1.6.1**. +It is autogenerated from the Rust config crate +([`nativelink-config/src`](https://github.com/TraceMachina/nativelink/tree/v1.6.1/nativelink-config/src)) via the `build-schema` binary, so +it can never drift from what the binary actually deserializes. + +## Top-level fields + +The root object (`CasConfig`) accepts the following fields: + +| Field | Type | Required | Description | +| --- | --- | --- | --- | +| `stores` | array of [NamedStoreConfig](#namedstoreconfig) | Yes | List of stores available to use in this config. The keys can be used in other configs when needing to reference a store. | +| `workers` | array of [WorkerConfig](#workerconfig) | — | Worker configurations used to execute jobs. | +| `schedulers` | array of [NamedSchedulerConfig](#namedschedulerconfig) | — | List of schedulers available to use in this config. The keys can be used in other configs when needing to reference a scheduler. | +| `servers` | array of [ServerConfig](#serverconfig) | Yes | Servers to setup for this process. | +| `experimental_origin_events` | [OriginEventsSpec](#origineventsspec) | — | Experimental - Origin events configuration. This is the service that will collect and publish nativelink events to a store for processing by an external service. | +| `global` | [GlobalConfig](#globalconfig) | — | Any global configurations that apply to all modules live here. | + +## Configuration types + +Every type reachable from the root configuration, in reading order. + +## NamedStoreConfig + +**Common fields** + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `name` | string | Yes | — | | + +Plus exactly one of the following variants (the key selects the variant): + +### `cache_metrics` + +Cache metrics store wraps another store and emits low-cardinality +OpenTelemetry cache operation metrics for the wrapped store. + +This wrapper is opt-in. Stores that are not explicitly wrapped by +`cache_metrics` are constructed exactly as they are without this +wrapper and do not pay its hot-path timing or recording cost. + +**Example JSON5 config:** +```json5 +"cache_metrics": { + "cache_type": "cas", + "backend": { + "filesystem": { + "content_path": "~/.cache/nativelink/content_path-cas", + "temp_path": "~/.cache/nativelink/tmp_path-cas" + } + } +} +``` + +**Type:** [CacheMetricsSpec](#cachemetricsspec) + +### `memory` + +Memory store will store all data in a hash map in memory. + +**Example JSON5 config:** +```json5 +"memory": { + "eviction_policy": { + "max_bytes": "10mb", + } +} +``` + +**Type:** [MemorySpec](#memoryspec) + +### `experimental_cloud_object_store` + +A generic blob store that will store files on the cloud +provider. This configuration will never delete files, so you are +responsible for purging old files in other ways. +It supports the following backends: + +1. **Amazon S3:** + S3 store will use Amazon's S3 service as a backend to store + the files. This configuration can be used to share files + across multiple instances. Uses system certificates for TLS + verification via `rustls-platform-verifier`. + + **Example JSON5 config:** + ```json5 + "experimental_cloud_object_store": { + "provider": "aws", + "region": "eu-north-1", + "bucket": "crossplane-bucket-af79aeca9", + "key_prefix": "test-prefix-index/", + "retry": { + "max_retries": 6, + "delay": 0.3, + "jitter": 0.5 + }, + "multipart_max_concurrent_uploads": 10 + } + ``` + +2. **Google Cloud Storage:** + GCS store uses Google's GCS service as a backend to store + the files. This configuration can be used to share files + across multiple instances. + + **Example JSON5 config:** + ```json5 + "experimental_cloud_object_store": { + "provider": "gcs", + "bucket": "test-bucket", + "key_prefix": "test-prefix-index/", + "retry": { + "max_retries": 6, + "delay": 0.3, + "jitter": 0.5 + }, + "multipart_max_concurrent_uploads": 10 + } + ``` + +3. **Azure Blob Store:** + Azure Blob store will use Microsoft's Azure Blob service as a + backend to store the files. This configuration can be used to + share files across multiple instances. + + **Example JSON5 config:** + ```json5 + "experimental_cloud_object_store": { + "provider": "azure", + "account_name": "cloudshell1393657559", + "container": "simple-test-container", + "key_prefix": "folder/", + "retry": { + "max_retries": 6, + "delay": 0.3, + "jitter": 0.5 + }, + "multipart_max_concurrent_uploads": 10 + } + ``` + +4. **NetApp ONTAP S3:** + NetApp ONTAP S3 store will use ONTAP's S3-compatible storage as a backend + to store files. This store is specifically configured for ONTAP's S3 requirements + including custom TLS configuration, credentials management, and proper vserver + configuration. + + This store uses AWS environment variables for credentials: + - `AWS_ACCESS_KEY_ID` + - `AWS_SECRET_ACCESS_KEY` + - `AWS_DEFAULT_REGION` + + **Example JSON5 config:** + ```json5 + "experimental_cloud_object_store": { + "provider": "ontap", + "endpoint": "https://ontap-s3-endpoint:443", + "vserver_name": "your-vserver", + "bucket": "your-bucket", + "root_certificates": "/path/to/certs.pem", // Optional + "key_prefix": "test-prefix/", // Optional + "retry": { + "max_retries": 6, + "delay": 0.3, + "jitter": 0.5 + }, + "multipart_max_concurrent_uploads": 10 + } + ``` + +5. **Cloudflare R2:** + R2 store uses Cloudflare's R2 service as a backend. R2 speaks the + S3 API, so this is a thin wrapper that derives the account-scoped + endpoint (`https://{account_id}.r2.cloudflarestorage.com`) for you. + + **Example JSON5 config:** + ```json5 + "experimental_cloud_object_store": { + "provider": "r2", + "account_id": "a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4", + "bucket": "nativelink-cas", + "key_prefix": "test-prefix/", + "retry": { + "max_retries": 6, + "delay": 0.3, + "jitter": 0.5 + }, + "multipart_max_concurrent_uploads": 10 + } + ``` + +6. **Oracle Cloud Infrastructure (OCI) Object Storage:** + OCI store uses Oracle Cloud Infrastructure's S3-compatible Object + Storage API. The path-style endpoint is derived from your Object + Storage `namespace` and `region` as + `https://{namespace}.compat.objectstorage.{region}.oci.customer-oci.com`. + Authenticate with a Customer Secret Key (Access Key/Secret Key pair + created under User Settings -> Customer secret keys in the OCI + console); the secret cannot be retrieved after generation, so read + it from an env var via shellexpand. + + **Example JSON5 config:** + ```json5 + "experimental_cloud_object_store": { + "provider": "oci", + "namespace": "your-object-storage-namespace", + "region": "us-phoenix-1", + "bucket": "nativelink-cas", + "access_key_id": "oci_access_key_id", + "secret_access_key": "oci_secret_access_key", + "key_prefix": "test-prefix/", + "retry": { + "max_retries": 6, + "delay": 0.3, + "jitter": 0.5 + } + } + ``` + +**Type:** [ExperimentalCloudObjectSpec](#experimentalcloudobjectspec) + +### `ontap_s3_existence_cache` + +ONTAP S3 Existence Cache provides a caching layer on top of the ONTAP S3 store +to optimize repeated existence checks. It maintains an in-memory cache of object +digests and periodically syncs this cache to disk for persistence. + +The cache helps reduce latency for repeated calls to check object existence, +while still ensuring eventual consistency with the underlying ONTAP S3 store. + +Example JSON5 config: +```json5 +"ontap_s3_existence_cache": { + "index_path": "/path/to/cache/index.json", + "sync_interval_seconds": 300, + "backend": { + "endpoint": "https://ontap-s3-endpoint:443", + "vserver_name": "your-vserver", + "bucket": "your-bucket", + "key_prefix": "test-prefix/" + } +} +``` + +**Type:** [OntapS3ExistenceCacheSpec](#ontaps3existencecachespec) + +### `verify` + +Verify store is used to apply verifications to an underlying +store implementation. It is strongly encouraged to validate +as much data as you can before accepting data from a client, +failing to do so may cause the data in the store to be +populated with invalid data causing all kinds of problems. + +The suggested configuration is to have the CAS validate the +hash and size and the AC validate nothing. + +**Example JSON5 config:** +```json5 +"verify": { + "backend": { + "memory": { + "eviction_policy": { + "max_bytes": "500mb" + } + }, + }, + "verify_size": true, + "verify_hash": true +} +``` + +**Type:** [VerifySpec](#verifyspec) + +### `completeness_checking` + +Completeness checking store verifies if the +output files & folders exist in the CAS before forwarding +the request to the underlying store. +Note: This store should only be used on AC stores. + +**Example JSON5 config:** +```json5 +"completeness_checking": { + "backend": { + "filesystem": { + "content_path": "~/.cache/nativelink/content_path-ac", + "temp_path": "~/.cache/nativelink/tmp_path-ac", + "eviction_policy": { + "max_bytes": "500mb", + } + } + }, + "cas_store": { + "ref_store": { + "name": "CAS_MAIN_STORE" + } + } +} +``` + +**Type:** [CompletenessCheckingSpec](#completenesscheckingspec) + +### `compression` + +A compression store that will compress the data inbound and +outbound. There will be a non-trivial cost to compress and +decompress the data, but in many cases if the final store is +a store that requires network transport and/or storage space +is a concern it is often faster and more efficient to use this +store before those stores. + +**Example JSON5 config:** +```json5 +"compression": { + "compression_algorithm": { + "lz4": {} + }, + "backend": { + "filesystem": { + "content_path": "/tmp/nativelink/data/content_path-cas", + "temp_path": "/tmp/nativelink/data/tmp_path-cas", + "eviction_policy": { + "max_bytes": "2gb", + } + } + } +} +``` + +**Type:** [CompressionSpec](#compressionspec) + +### `dedup` + +A dedup store will take the inputs and run a rolling hash +algorithm on them to slice the input into smaller parts then +run a sha256 algorithm on the slice and if the object doesn't +already exist, upload the slice to the `content_store` using +a new digest of just the slice. Once all parts exist, an +Action-Cache-like digest will be built and uploaded to the +`index_store` which will contain a reference to each +chunk/digest of the uploaded file. Downloading a request will +first grab the index from the `index_store`, and forward the +download content of each chunk as if it were one file. + +This store is exceptionally good when the following conditions +are met: +* Content is mostly the same (inserts, updates, deletes are ok) +* Content is not compressed or encrypted +* Uploading or downloading from `content_store` is the bottleneck. + +Note: This store pairs well when used with `CompressionSpec` as +the `content_store`, but never put `DedupSpec` as the backend of +`CompressionSpec` as it will negate all the gains. + +Note: When running `.has()` on this store, it will only check +to see if the entry exists in the `index_store` and not check +if the individual chunks exist in the `content_store`. + +**Example JSON5 config:** +```json5 +"dedup": { + "index_store": { + "memory": { + "eviction_policy": { + "max_bytes": "1GB", + } + } + }, + "content_store": { + "compression": { + "compression_algorithm": { + "lz4": {} + }, + "backend": { + "fast_slow": { + "fast": { + "memory": { + "eviction_policy": { + "max_bytes": "500MB", + } + } + }, + "slow": { + "filesystem": { + "content_path": "/tmp/nativelink/data/content_path-content", + "temp_path": "/tmp/nativelink/data/tmp_path-content", + "eviction_policy": { + "max_bytes": "2gb" + } + } + } + } + } + } + } +} +``` + +**Type:** [DedupSpec](#dedupspec) + +### `existence_cache` + +Existence store will wrap around another store and cache calls +to has so that subsequent `has_with_results` calls will be +faster. This is useful for cases when you have a store that +is slow to respond to has calls. +Note: This store should only be used on CAS stores. + +**Example JSON5 config:** +```json5 +"existence_cache": { + "backend": { + "memory": { + "eviction_policy": { + "max_bytes": "500mb", + } + } + }, + // Note this is the existence store policy, not the backend policy + "eviction_policy": { + "max_seconds": 100, + } +} +``` + +**Type:** [ExistenceCacheSpec](#existencecachespec) + +### `fast_slow` + +`FastSlow` store will first try to fetch the data from the `fast` +store and then if it does not exist try the `slow` store. +When the object does exist in the `slow` store, it will copy +the data to the `fast` store while returning the data. +This store should be thought of as a store that "buffers" +the data to the `fast` store. +On uploads it will mirror data to both `fast` and `slow` stores. + +WARNING: If you need data to always exist in the `slow` store +for something like remote execution, be careful because this +store will never check to see if the objects exist in the +`slow` store if it exists in the `fast` store (i.e., it assumes +that if an object exists in the `fast` store it will exist in +the `slow` store). + +***Example JSON5 config:*** +```json5 +"fast_slow": { + "fast": { + "filesystem": { + "content_path": "/tmp/nativelink/data/content_path-index", + "temp_path": "/tmp/nativelink/data/tmp_path-index", + "eviction_policy": { + "max_bytes": "500mb", + } + } + }, + "slow": { + "filesystem": { + "content_path": "/tmp/nativelink/data/content_path-index", + "temp_path": "/tmp/nativelink/data/tmp_path-index", + "eviction_policy": { + "max_bytes": "500mb", + } + } + } +} +``` + +**Type:** [FastSlowSpec](#fastslowspec) + +### `shard` + +Shards the data to multiple stores. This is useful for cases +when you want to distribute the load across multiple stores. +The digest hash is used to determine which store to send the +data to. + +**Example JSON5 config:** +```json5 +"shard": { + "stores": [ + { + "store": { + "memory": { + "eviction_policy": { + "max_bytes": "10mb" + }, + }, + }, + "weight": 1 + }] +} +``` + +**Type:** [ShardSpec](#shardspec) + +### `filesystem` + +Stores the data on the filesystem. This store is designed for +local persistent storage. Restarts of this program should restore +the previous state, meaning anything uploaded will be persistent +as long as the filesystem integrity holds. + +**Example JSON5 config:** +```json5 +"filesystem": { + "content_path": "/tmp/nativelink/data-worker-test/content_path-cas", + "temp_path": "/tmp/nativelink/data-worker-test/tmp_path-cas", + "eviction_policy": { + "max_bytes": "10gb", + } +} +``` + +**Type:** [FilesystemSpec](#filesystemspec) + +### `ref_store` + +Store used to reference a store in the root store manager. +This is useful for cases when you want to share a store in different +nested stores. Example, you may want to share the same memory store +used for the action cache, but use a `FastSlowSpec` and have the fast +store also share the memory store for efficiency. + +**Example JSON5 config:** +```json5 +"ref_store": { + "name": "FS_CONTENT_STORE" +} +``` + +**Type:** [RefSpec](#refspec) + +### `size_partitioning` + +Uses the size field of the digest to separate which store to send the +data. This is useful for cases when you'd like to put small objects +in one store and large objects in another store. This should only be +used if the size field is the real size of the content, in other +words, don't use on AC (Action Cache) stores. Any store where you can +safely use `VerifySpec.verify_size = true`, this store should be safe +to use (i.e., CAS stores). + +**Example JSON5 config:** +```json5 +"size_partitioning": { + "size": "128mib", + "lower_store": { + "memory": { + "eviction_policy": { + "max_bytes": "${NATIVELINK_CAS_MEMORY_CONTENT_LIMIT:-100mb}" + } + } + }, + "upper_store": { + /// This store discards data larger than 128mib. + "noop": {} + } +} +``` + +**Type:** [SizePartitioningSpec](#sizepartitioningspec) + +### `grpc` + +This store will pass-through calls to another GRPC store. This store +is not designed to be used as a sub-store of another store, but it +does satisfy the interface and will likely work. + +One major GOTCHA is that some stores use a special function on this +store to get the size of the underlying object, which is only reliable +when this store is serving the a CAS store, not an AC store. If using +this store directly without being a child of any store there are no +side effects and is the most efficient way to use it. + +**Example JSON5 config:** +```json5 +"grpc": { + "instance_name": "main", + "endpoints": [ + {"address": "grpc://${CAS_ENDPOINT:-127.0.0.1}:50051"} + ], + "connections_per_endpoint": "5", + "rpc_timeout_s": "5m", + "store_type": "ac", + // Static headers attached to every outgoing request to the upstream + // remote cache. Useful for fixed service-account credentials. + "headers": { + "authorization": "Bearer my-static-token" + }, + // Header names to copy from the inbound client request and forward to + // the upstream remote cache. Use this to pass through dynamic + // credentials such as a JWT sent by the build client. + "forward_headers": ["authorization", "x-custom-token"] +} +``` + +**Type:** [GrpcSpec](#grpcspec) + +### `redis_store` + +Stores data in any stores compatible with Redis APIs. + +Pairs well with `SizePartitioning` and/or `FastSlow` stores. +Ideal for accepting small object sizes as most Redis store +services have a max file upload of between 256Mb-512Mb. + +**Example JSON5 config:** +```json5 +"redis_store": { + "addresses": [ + "redis://127.0.0.1:6379/", + ], + "max_client_permits": 1000, +} +``` + +**Type:** [RedisSpec](#redisspec) + +### `noop` + +Noop store is a store that sends streams into the void and all data +retrieval will return 404 (`NotFound`). This can be useful for cases +where you may need to partition your data and part of your data needs +to be discarded. + +**Example JSON5 config:** +```json5 +"noop": {} +``` + +**Type:** [NoopSpec](#noopspec) + +### `experimental_mongo` + +Experimental `MongoDB` store implementation. + +This store uses `MongoDB` as a backend for storing data. It supports +both CAS (Content Addressable Storage) and scheduler data with +optional change streams for real-time updates. + +**Example JSON5 config:** +```json5 +"experimental_mongo": { + "connection_string": "mongodb://localhost:27017", + "database": "nativelink", + "cas_collection": "cas", + "key_prefix": "cas:", + "read_chunk_size": 65536, + "max_concurrent_uploads": 10, + "enable_change_streams": false, + "max_requests": "100" +} +``` + +**Type:** [ExperimentalMongoSpec](#experimentalmongospec) + +## WorkerConfig + +Plus exactly one of the following variants (the key selects the variant): + +### `local` + +A worker type that executes jobs locally on this machine. + +**Type:** [LocalWorkerConfig](#localworkerconfig) + +## NamedSchedulerConfig + +**Common fields** + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `name` | string | Yes | — | | + +Plus exactly one of the following variants (the key selects the variant): + +### `simple` + +**Type:** [SimpleSpec](#simplespec) + +### `grpc` + +**Type:** [SchedulerGrpcSpec](#schedulergrpcspec) + +### `cache_lookup` + +**Type:** [CacheLookupSpec](#cachelookupspec) + +### `property_modifier` + +**Type:** [PropertyModifierSpec](#propertymodifierspec) + +### `historical_resource` + +**Type:** [HistoricalResourceSpec](#historicalresourcespec) + +## ServerConfig + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `name` | string | — | {index of server in config} | Name of the server. This is used to help identify the service for telemetry and logs. | +| `listener` | [ListenerConfig](#listenerconfig) | Yes | — | Configuration | +| `services` | [ServicesConfig](#servicesconfig) | — | — | Services to attach to server. | +| `experimental_identity_header` | [IdentityHeaderSpec](#identityheaderspec) | — | {see `IdentityHeaderSpec`} | The config related to identifying the client. | + +## OriginEventsSpec + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `publisher` | [OriginEventsPublisherSpec](#origineventspublisherspec) | Yes | — | The publisher configuration for origin events. | +| `max_event_queue_size` | integer (uint) | — | 65536 (zero defaults to this) | The maximum number of events to queue before applying back pressure. IMPORTANT: Backpressure causes all clients to slow down significantly. Zero is default. | + +## GlobalConfig + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `max_open_files` | integer (uint) | Yes | 24576 (= 24 * 1024) | Maximum number of open files that can be opened at one time. This value is not strictly enforced, it is a best effort. Some internal libraries open files or read metadata from a files which do not obey this limit, however the vast majority of cases will have this limit be honored. This value must be larger than `ulimit -n` to have any effect. Any network open file descriptors is not counted in this limit, but is counted in the kernel limit. It is a good idea to set a very large `ulimit -n`. Note: This value must be greater than 10. | +| `default_digest_hash_function` | [ConfigDigestHashFunction](#configdigesthashfunction) | — | `ConfigDigestHashFunction::sha256` | Default hash function to use while uploading blobs to the CAS when not set by client. | +| `default_digest_size_health_check` | integer (uint) | — | 1024*1024 (1MiB) | Default digest size to use for health check when running diagnostics checks. Health checks are expected to use this size for filling a buffer that is used for creation of digest. | + +## CacheMetricsSpec + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `cache_type` | string | Yes | — | Low-cardinality cache type label for metrics, for example `cas` or `ac`. | +| `backend` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | Store to wrap with cache operation metrics. | + +## MemorySpec + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `eviction_policy` | [EvictionPolicy](#evictionpolicy) | — | — | Policy used to evict items out of the store. Failure to set this value will cause items to never be removed from the store causing infinite memory usage. | + +## ExperimentalCloudObjectSpec + +See [`experimental_cloud_object_store`](#experimental_cloud_object_store-1) for details + +## OntapS3ExistenceCacheSpec + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `index_path` | string | Yes | — | | +| `sync_interval_seconds` | integer (uint32) | Yes | — | | +| `backend` | [ExperimentalOntapS3Spec](#experimentalontaps3spec) | Yes | — | | + +## VerifySpec + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `backend` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | The underlying store wrap around. All content will first flow through self before forwarding to backend. In the event there is an error detected in self, the connection to the backend will be terminated, and early termination should always cause updates to fail on the backend. | +| `verify_size` | boolean | — | `false` | If set the store will verify the size of the data before accepting an upload of data. | +| `verify_hash` | boolean | — | `false` | If the data should be hashed and verify that the key matches the computed hash. The hash function is automatically determined based request and if not set will use the global default. | + +## CompletenessCheckingSpec + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `backend` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | The underlying store that will have it's results validated before sending to client. | +| `cas_store` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | When a request is made, the results are decoded and all output digests/files are verified to exist in this CAS store before returning success. | + +## CompressionSpec + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `backend` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | The underlying store wrap around. All content will first flow through self before forwarding to backend. In the event there is an error detected in self, the connection to the backend will be terminated, and early termination should always cause updates to fail on the backend. | +| `compression_algorithm` | [CompressionAlgorithm](#compressionalgorithm) | Yes | — | The compression algorithm to use. | + +## DedupSpec + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `index_store` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | Store used to store the index of each dedup slice. This store should generally be fast and small. | +| `content_store` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | The store where the individual chunks will be uploaded. This store should generally be the slower & larger store. | +| `min_size` | integer (uint32) | — | 64k | Minimum size that a chunk will be when slicing up the content. Note: This setting can be increased to improve performance because it will actually not check this number of bytes when deciding where to partition the data. | +| `normal_size` | integer (uint32) | — | 256k | A best-effort attempt will be made to keep the average size of the chunks to this number. It is not a guarantee, but a slight attempt will be made. | +| `max_size` | integer (uint32) | — | 512k | Maximum size a chunk is allowed to be. | +| `max_concurrent_fetch_per_get` | integer (uint32) | — | 10 | Due to implementation detail, we want to prefer to download the first chunks of the file so we can stream the content out and free up some of our buffers. This configuration will be used to restrict the number of concurrent chunk downloads at a time per `get()` request. | + +## ExistenceCacheSpec + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `backend` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | The underlying store wrap around. All content will first flow through self before forwarding to backend. In the event there is an error detected in self, the connection to the backend will be terminated, and early termination should always cause updates to fail on the backend. | +| `eviction_policy` | [EvictionPolicy](#evictionpolicy) | — | — | Policy used to evict items out of the store. Failure to set this value will cause items to never be removed from the store causing infinite memory usage. | + +## FastSlowSpec + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `fast` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | Fast store that will be attempted to be contacted before reaching out to the `slow` store. | +| `fast_direction` | [StoreDirection](#storedirection) | — | `"both"` | How to handle the fast store. This can be useful to set to Get for worker nodes such that results are persisted to the slow store only. | +| `slow` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | If the object does not exist in the `fast` store it will try to get it from this store. | +| `slow_direction` | [StoreDirection](#storedirection) | — | `"both"` | How to handle the slow store. This can be useful if creating a diode and you wish to have an upstream read only store. | +| `bypass_dedup_threshold_bytes` | integer (uint64) | — | disabled (0) | Reads of blobs at or above this size skip the leader/follower dedup map and stream straight from the slow store without populating the fast tier. `0` (the default) disables the bypass: every read goes through dedup, matching the prior behaviour. Enable it by setting a threshold — 256 MiB is a reasonable starting point for backends where large-blob dedup is a net loss (followers tend to time out anyway), but the right value is workload-dependent. | + +## ShardSpec + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `stores` | array of [ShardConfig](#shardconfig) | Yes | — | Stores to shard the data to. | + +## FilesystemSpec + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `content_path` | string | Yes | — | Path on the system where to store the actual content. This is where the bulk of the data will be placed. On service startup this folder will be scanned and all files will be added to the cache. In the event one of the files doesn't match the criteria, the file will be deleted. | +| `temp_path` | string | Yes | — | A temporary location of where files that are being uploaded or deleted will be placed while the content cannot be guaranteed to be accurate. This location must be on the same block device as `content_path` so atomic moves can happen (i.e., move without copy). All files in this folder will be deleted on every startup. | +| `read_buffer_size` | integer (uint32) | — | 32k | Buffer size to use when reading files. Generally this should be left to the default value except for testing. | +| `eviction_policy` | [EvictionPolicy](#evictionpolicy) | — | — | Policy used to evict items out of the store. Failure to set this value will cause items to never be removed from the store causing infinite memory usage. | +| `block_size` | integer (uint64) | — | 4kb | The block size of the filesystem for the running machine value is used to determine an entry's actual size on disk consumed For a 4KB block size filesystem, a 1B file actually consumes 4KB | +| `max_concurrent_writes` | integer (uint) | — | unlimited | Maximum number of concurrent write operations allowed. Each write involves streaming data to a temp file and calling `sync_all()`, which can saturate disk I/O when many writes happen simultaneously. Limiting concurrency prevents disk saturation from blocking the async runtime. A value of 0 means unlimited (no concurrency limit). | + +## RefSpec + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `name` | string | Yes | — | Name of the store under the root "stores" config object. | + +## SizePartitioningSpec + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `size` | integer (uint64) | Yes | — | Size to partition the data on. | +| `lower_store` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | Store to send data when object is < (less than) size. | +| `upper_store` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | Store to send data when object is >= (less than eq) size. | + +## GrpcSpec + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `instance_name` | string | — | `""` | Instance name for GRPC calls. Proxy calls will have the `instance_name` changed to this. | +| `endpoints` | array of [GrpcEndpoint](#grpcendpoint) | Yes | — | The endpoint of the grpc connection. | +| `store_type` | [StoreType](#storetype) | Yes | — | The type of the upstream store, this ensures that the correct server calls are made. | +| `retry` | [Retry](#retry) | — | — | Retry configuration to use when a network request fails. | +| `max_concurrent_requests` | integer (uint) | — | `0` | Limit the number of simultaneous upstream requests to this many. A value of zero is treated as unlimited. If the limit is reached the request is queued. | +| `connections_per_endpoint` | integer (uint) | — | `0` | The number of connections to make to each specified endpoint to balance the load over multiple TCP connections. Default 1. | +| `rpc_timeout_s` | integer (uint64) | — | 0 (disabled) | Maximum time (seconds) allowed for a single RPC request (e.g. a `ByteStream.Write` call) before it is cancelled. | +| `use_legacy_resource_names` | boolean | — | false | Use legacy `ByteStream` resource name format, omitting the digest function component from the path. | +| `headers` | map of string to string | — | — | Static headers to attach to every outgoing gRPC request sent to this store's upstream endpoints. Useful for fixed authentication tokens (e.g. `{"authorization": "Bearer "}`) and other static metadata. | +| `forward_headers` | array of string | — | — | Header names to forward from the incoming client request to every outgoing upstream request. The header value is taken from the client request that triggered this store operation. Use this to pass through dynamic credentials such as JWT tokens sent by build clients. | + +## RedisSpec + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `addresses` | array of string | Yes | — | The hostname or IP address of the Redis server. Ex: `["redis://username:password@redis-server-url:6380/99"]` 99 Represents database ID, 6380 represents the port. | +| `response_timeout_s` | integer (uint64) | — | 10 | DEPRECATED: use `command_timeout_ms` The response timeout for the Redis connection in seconds. | +| `connection_timeout_s` | integer (uint64) | — | 10 | DEPRECATED: use `connection_timeout_ms` | +| `experimental_pub_sub_channel` | string | — | (Empty String / No Channel) | An optional and experimental Redis channel to publish write events to. | +| `key_prefix` | string | — | (Empty String / No Prefix) | An optional prefix to prepend to all keys in this store. | +| `mode` | [RedisMode](#redismode) | — | standard, | Set the mode Redis is operating in. | +| `broadcast_channel_capacity` | integer (uint) | — | `0` | Deprecated as redis-rs doesn't use it | +| `command_timeout_ms` | integer (uint64) | — | 10000 (10 seconds) | The amount of time in milliseconds until the Redis store considers the command to be timed out. This will trigger a retry of the command and potentially a reconnection to the Redis server. | +| `connection_timeout_ms` | integer (uint64) | — | 3000 (3 seconds) | The amount of time in milliseconds until the Redis store considers the connection to unresponsive. This will trigger a reconnection to the Redis server. | +| `health_check_timeout_ms` | integer (uint64) | — | 4000 (4 seconds) | Per-call ceiling for the `check_health` PING in milliseconds. | +| `read_chunk_size` | integer (uint) | — | 64KiB | The amount of data to read from the Redis server at a time. This is used to limit the amount of memory used when reading large objects from the Redis server as well as limiting the amount of time a single read operation can take. | +| `connection_pool_size` | integer (uint) | — | 3 | The number of connections to keep open to the Redis servers. | +| `max_chunk_uploads_per_update` | integer (uint) | — | 10 | The maximum number of upload chunks to allow per update. This is used to limit the amount of memory used when uploading large objects to the Redis server. A good rule of thumb is to think of the data as: `AVAIL_MEMORY / (read_chunk_size * max_chunk_uploads_per_update) = THORETICAL_MAX_CONCURRENT_UPLOADS` (note: it is a good idea to divide `AVAIL_MAX_MEMORY` by ~10 to account for other memory usage) | +| `scan_count` | integer (uint) | — | 10000 | The COUNT value passed when scanning keys in Redis. This is used to hint the amount of work that should be done per response. | +| `retry` | [Retry](#retry) | — | — | Retry configuration to use when a network request fails. | +| `max_client_permits` | integer (uint) | — | 500 | Maximum number of permitted actions to the Redis store at any one time This stops problems with timeouts due to many, many inflight actions | +| `max_count_per_cursor` | integer (uint64) | — | 1500 | Maximum number of items returned per cursor for the search indexes May reduce thundering herd issues with worker provisioner at higher node counts, | + +## NoopSpec + +_No fields._ + +## ExperimentalMongoSpec + +Configuration for `ExperimentalMongoDB` store. + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `connection_string` | string | Yes | — | `ExperimentalMongoDB` connection string. Example: <mongodb://localhost:27017> or <mongodb+srv://cluster.mongodb.net> | +| `database` | string | — | "nativelink" | The database name to use. | +| `cas_collection` | string | — | "cas" | The collection name for CAS data. | +| `scheduler_collection` | string | — | "scheduler" | The collection name for scheduler data. | +| `key_prefix` | string | — | "" | Prefix to prepend to all keys stored in `MongoDB`. | +| `read_chunk_size` | integer (uint) | — | 65536 (64KB) | The maximum amount of data to read from `MongoDB` in a single chunk (in bytes). | +| `max_concurrent_uploads` | integer (uint) | — | 10 | Deprecated, unused Maximum number of concurrent uploads allowed. | +| `connection_timeout_ms` | integer (uint64) | — | 3000 | Connection timeout in milliseconds. | +| `command_timeout_ms` | integer (uint64) | — | 10000 | Command timeout in milliseconds. | +| `enable_change_streams` | boolean | — | false | Enable `MongoDB` change streams for real-time updates. Required for scheduler subscriptions. | +| `write_concern_w` | string | — | — | Write concern 'w' parameter. Can be a number (e.g., 1) or string (e.g., "majority"). | +| `write_concern_j` | boolean | — | — | Write concern 'j' parameter (journal acknowledgment). | +| `write_concern_timeout_ms` | integer (uint32) | — | — | Write concern timeout in milliseconds. | +| `max_requests` | integer (uint) | — | Unlimited | Limits the number of requests at any one time | + +## LocalWorkerConfig + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `name` | string | — | {Index position in the workers list} | Name of the worker. This is give a more friendly name to a worker for logging and metric publishing. This is also the prefix of the worker id (i.e., "{name}{uuidv6}"). | +| `worker_api_endpoint` | [EndpointConfig](#endpointconfig) | Yes | — | Endpoint which the worker will connect to the scheduler's `WorkerApiService`. | +| `max_action_timeout_s` | integer (uint) | — | 20 minutes | The maximum time an action is allowed to run. If a task requests for a timeout longer than this time limit, the task will be rejected. Value in seconds. | +| `max_upload_timeout_s` | integer (uint) | — | 10 minutes | Maximum time allowed for uploading action results to CAS after execution completes. If upload takes longer than this, the action fails with `DeadlineExceeded` and may be retried by the scheduler. Value in seconds. | +| `max_cleanup_wait_s` | integer (uint) | — | 30 seconds | Maximum time to wait for action directory cleanup before timing out. Value in seconds. | +| `max_cleanup_backoff_ms` | integer (uint) | — | 500 milliseconds | Maximum backoff duration for exponential backoff when waiting for cleanup. Value in milliseconds. | +| `max_inflight_tasks` | integer (uint64) | — | 0 (infinite tasks) | Maximum number of inflight tasks this worker can cope with. | +| `timeout_handled_externally` | boolean | — | false (`NativeLink` fully handles timeouts) | If timeout is handled in `entrypoint` or another wrapper script. If set to true `NativeLink` will not honor the timeout the action requested and instead will always force kill the action after `max_action_timeout` has been reached. If this is set to false, the smaller value of the action's timeout and `max_action_timeout` will be used to which `NativeLink` will kill the action. | +| `entrypoint` | string | — | {Use the command from the job request} | The command to execute on every execution request. This will be parsed as a command + arguments (not shell). Example: "run.sh" and a job with command: "sleep 5" will result in a command like: "run.sh sleep 5". | +| `experimental_precondition_script` | string | — | — | An optional script to run before every action is processed on the worker. The value should be the full path to the script to execute and will pause all actions on the worker if it returns an exit code other than 0. If not set, then the worker will never pause and will continue to accept jobs according to the scheduler configuration. This is useful, for example, if the worker should not take any more actions until there is enough resource available on the machine to handle them. | +| `cas_fast_slow_store` | string | Yes | — | Underlying CAS store that the worker will use to download CAS artifacts. This store must be a `FastSlowStore`. The `fast` store must be a `FileSystemStore` because it will use hardlinks when building out the files instead of copying the files. The slow store must eventually resolve to the same store the scheduler/client uses to send job requests. | +| `upload_action_result` | [UploadActionResultConfig](#uploadactionresultconfig) | — | — | Configuration for uploading action results. | +| `work_directory` | string | Yes | — | The directory work jobs will be executed from. This directory will be fully managed by the worker service and will be purged on startup. This directory and the directory referenced in `local_filesystem_store_ref`'s `stores::FilesystemStore::content_path` must be on the same filesystem. Hardlinks will be used when placing files that are accessible to the jobs that are sourced from `local_filesystem_store_ref`'s `content_path`. | +| `platform_properties` | map of string to [WorkerProperty](#workerproperty) | Yes | — | Properties of this worker. This configuration will be sent to the scheduler and used to tell the scheduler to restrict what should be executed on this worker. | +| `additional_environment` | map of string to [EnvironmentSource](#environmentsource) | — | — | An optional mapping of environment names to set for the execution as well as those specified in the action itself. If set, will set each key as an environment variable before executing the job with the value of the environment variable being the value of the property of the action being executed of that name or the fixed value. | +| `directory_cache` | [DirectoryCacheConfig](#directorycacheconfig) | — | — | Optional directory cache configuration for improving performance by caching reconstructed input directories and using hardlinks instead of rebuilding them from CAS for every action. | +| `use_namespaces` | boolean | — | False | Whether to use namespaces to isolate the execution. This is only available on Linux. It is highly recommended as it avoids a number of issues with zombie processes and also provides additional hermeticity. If explicitly set to true and it is not supported the worker will exit with an error. | +| `use_mount_namespace` | boolean | — | False | Whether to use a mount namespace to isolate the worker root. This is only available on Linux and when `use_namespaces` is true. It is highly recommended provides additional hermeticity. If explicitly set to true and it is not supported or `use_namespaces` is not set to true the worker will exit with an error. | + +## SimpleSpec + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `supported_platform_properties` | map of string to [PropertyType](#propertytype) | — | — | A list of supported platform properties mapped to how these properties are used when the scheduler looks for worker nodes capable of running the task. | +| `retain_completed_for_s` | integer (uint32) | — | 60 seconds | The amount of time to retain completed actions for in case a `WaitExecution` is called after the action has completed. | +| `client_action_timeout_s` | integer (uint64) | — | 60 seconds | Mark operations as completed with error if no client has updated them within this duration. | +| `worker_timeout_s` | integer (uint64) | — | 5 seconds | Remove workers from pool once the worker has not responded in this amount of time in seconds. | +| `max_action_executing_timeout_s` | integer (uint64) | — | 0 (disabled) | Maximum time (seconds) an action can stay in Executing state without any worker update before being timed out and re-queued. This applies regardless of worker keepalive status, catching cases where a worker is alive (sending keepalives) but stuck on a specific action. Set to 0 to disable (relies only on `worker_timeout_s`). | +| `max_job_retries` | integer (uint) | — | 3 | If a job returns an internal error or times out this many times when attempting to run on a worker the scheduler will return the last error to the client. Jobs will be retried and this configuration is to help prevent one rogue job from infinitely retrying and taking up a lot of resources when the task itself is the one causing the server to go into a bad state. | +| `allocation_strategy` | [WorkerAllocationStrategy](#workerallocationstrategy) | — | `"least_recently_used"` | The strategy used to assign workers jobs. | +| `experimental_backend` | [ExperimentalSimpleSchedulerBackend](#experimentalsimpleschedulerbackend) | — | memory | The storage backend to use for the scheduler. | +| `worker_match_logging_interval_s` | integer (int64) | — | `10` | Every N seconds, do logging of worker matching e.g. "worker busy", "can't find any worker" Defaults to 10s. Can be set to `-1` to disable | + +## SchedulerGrpcSpec + +A scheduler that forwards requests to an upstream scheduler. This +is useful to use when doing some kind of local action cache or CAS away from +the main cluster of workers. In general, it's more efficient to point the +build at the main scheduler directly though. + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `endpoint` | [GrpcEndpoint](#grpcendpoint) | Yes | — | The upstream scheduler to forward requests to. | +| `retry` | [Retry](#retry) | — | — | Retry configuration to use when a network request fails. | +| `max_concurrent_requests` | integer (uint) | — | unlimited | Limit the number of simultaneous upstream requests to this many. A value of zero is treated as unlimited. If the limit is reached the request is queued. | +| `connections_per_endpoint` | integer (uint) | — | 1 | The number of connections to make to each specified endpoint to balance the load over multiple TCP connections. | + +## CacheLookupSpec + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `ac_store` | string | Yes | — | The reference to the action cache store used to return cached actions from rather than running them again. To prevent unintended issues, this store should probably be a `CompletenessCheckingSpec`. | +| `scheduler` | [SchedulerSpec](#schedulerspec) | Yes | — | The nested scheduler to use if cache lookup fails. | + +## PropertyModifierSpec + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `modifications` | array of [PropertyModification](#propertymodification) | Yes | — | A list of modifications to perform to incoming actions for the nested scheduler. These are performed in order and blindly, so removing a property that doesn't exist is fine and overwriting an existing property is also fine. If adding properties that do not exist in the nested scheduler is not supported and will likely cause unexpected behaviour. | +| `scheduler` | [SchedulerSpec](#schedulerspec) | Yes | — | The nested scheduler to use after modifying the properties. | + +## HistoricalResourceSpec + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `hints_file` | string | Yes | — | JSON file containing historical resource hints keyed by Bazel `RequestMetadata` `target_id` and/or `action_mnemonic`. | +| `refresh_interval_s` | integer (uint64) | — | 30 seconds | Reload interval for `hints_file`. Set to 0 to load once. | +| `cpu_property_name` | string | — | `cpu_count` | Platform property name used for CPU minimum values. | +| `memory_property_name` | string | — | `memory_kb` | Platform property name used for memory minimum values, expressed in KiB. | +| `scheduler` | [SchedulerSpec](#schedulerspec) | Yes | — | The nested scheduler to use after applying resource hints. | + +## ListenerConfig + +Plus exactly one of the following variants (the key selects the variant): + +### `http` + +Listener for HTTP/HTTPS/HTTP2 sockets. + +**Type:** [HttpListener](#httplistener) + +## ServicesConfig + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `cas` | array of [CasServiceConfig](#casserviceconfig) | — | — | The Content Addressable Storage (CAS) backend config. The key is the `instance_name` used in the protocol and the value is the underlying CAS store config. | +| `ac` | array of [ActionCacheServiceConfig](#actioncacheserviceconfig) | — | — | The Action Cache (AC) backend config. The key is the `instance_name` used in the protocol and the value is the underlying AC store config. | +| `capabilities` | array of [CapabilitiesServiceConfig](#capabilitiesserviceconfig) | — | — | Capabilities service is required in order to use most of the bazel protocol. This service is used to provide the supported features and versions of this bazel GRPC service. | +| `execution` | array of [ExecutionServiceConfig](#executionserviceconfig) | — | — | The remote execution service configuration. NOTE: This service is under development and is currently just a place holder. | +| `bytestream` | array of [ByteStreamServiceConfig](#bytestreamserviceconfig) | — | — | This is the service used to stream data to and from the CAS. Bazel's protocol strongly encourages users to use this streaming interface to interact with the CAS when the data is large. | +| `fetch` | array of [FetchServiceConfig](#fetchserviceconfig) | — | — | These two are collectively the Remote Asset protocol, but it's defined as two separate services | +| `push` | array of [PushServiceConfig](#pushserviceconfig) | — | — | | +| `worker_api` | [WorkerApiConfig](#workerapiconfig) | — | — | This is the service used for workers to connect and communicate through. NOTE: This service should be served on a different, non-public port. In other words, `worker_api` configuration should not have any other services that are served on the same port. Doing so is a security risk, as workers have a different permission set than a client that makes the remote execution/cache requests. | +| `experimental_bep` | [BepConfig](#bepconfig) | — | — | Experimental - Build Event Protocol (BEP) configuration. This is the service that will consume build events from the client and publish them to a store for processing by an external service. | +| `admin` | [AdminConfig](#adminconfig) | — | — | This is the service for any administrative tasks. It provides a REST API endpoint for administrative purposes. | +| `health` | [HealthConfig](#healthconfig) | — | — | This is the service for health status check. | + +## IdentityHeaderSpec + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `header_name` | string | — | "x-identity" | The name of the header to look for the identity in. | +| `required` | boolean | — | `false` | If the header is required to be set or fail the request. | + +## OriginEventsPublisherSpec + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `store` | string | Yes | — | The store to publish nativelink events to. The store name referenced in the `stores` map in the main config. | + +## ConfigDigestHashFunction + +| Value | Description | +| --- | --- | +| `"sha256"` | Use the sha256 hash function. [https://en.wikipedia.org/wiki/SHA-2](https://en.wikipedia.org/wiki/SHA-2) | +| `"blake3"` | Use the blake3 hash function. [https://en.wikipedia.org/wiki/BLAKE_(hash_function)](https://en.wikipedia.org/wiki/BLAKE_(hash_function)) | + +## EvictionPolicy + +Eviction policy always works on LRU (Least Recently Used). Any time an entry +is touched it updates the timestamp. Inserts and updates will execute the +eviction policy removing any expired entries and/or the oldest entries +until the store size becomes smaller than `max_bytes`. + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `max_bytes` | integer (uint) | — | 0 | Maximum number of bytes before eviction takes place. Zero means never evict based on size. | +| `evict_bytes` | integer (uint) | — | 0 | When eviction starts based on hitting `max_bytes`, continue until `max_bytes - evict_bytes` is met to create a low watermark. This stops operations from thrashing when the store is close to the limit. | +| `max_seconds` | integer (uint32) | — | 0 | Maximum number of seconds for an entry to live since it was last accessed before it is evicted. Zero means never evict based on time. | +| `max_count` | integer (uint64) | — | 0 | Maximum size of the store before an eviction takes place. Zero means never evict based on count. | + +## Retry + +Retry configuration. This configuration is exponential and each iteration +a jitter as a percentage is applied of the calculated delay. For example: +```haskell +Retry{ + max_retries: 7, + delay: 0.1, + jitter: 0.5, +} +``` +will result in: + +| Attempt | Delay | +| --- | --- | +| 1 | 0 ms | +| 2 | 75 to 125 ms | +| 3 | 150 to 250 ms | +| 4 | 300 to 500 ms | +| 5 | 600 ms to 1 s | +| 6 | 1.2 to 2 s | +| 7 | 2.4 to 4 s | +| 8 | 4.8 to 8 s | + +The total delay is additive, so this example produces 9.525 to 15.875 s of total delay for a single request. + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `max_retries` | integer (uint) | — | `0` | Maximum number of retries until retrying stops. Setting this to zero will always attempt 1 time, but not retry. | +| `delay` | number (float) | — | `0` | Delay in seconds for exponential back off. | +| `jitter` | number (float) | — | `0` | Amount of jitter to add as a percentage in decimal form. This will change the formula like: | +| `retry_on_errors` | array of [ErrorCode](#errorcode) | — | — | A list of error codes to retry on, if this is not set then the default error codes to retry on are used. These default codes are the most likely to be non-permanent: `Unknown`; `Cancelled`; `DeadlineExceeded`; `ResourceExhausted`; `Aborted`; `Internal`; `Unavailable`; `DataLoss` | + +## ExperimentalOntapS3Spec + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `endpoint` | string | Yes | — | | +| `vserver_name` | string | Yes | — | | +| `bucket` | string | Yes | — | | +| `root_certificates` | string | — | — | | +| `key_prefix` | string | — | — | If you wish to prefix the location in the bucket. If None, no prefix will be used. | +| `retry` | [Retry](#retry) | — | — | Retry configuration to use when a network request fails. | +| `consider_expired_after_s` | integer (uint32) | — | 0 | If the number of seconds since the `last_modified` time of the object is greater than this value, the object will not be considered "existing". This allows for external tools to delete objects that have not been uploaded in a long time. If a client receives a `NotFound` the client should re-upload the object. | +| `max_retry_buffer_per_request` | integer (uint) | — | 5MB | The maximum buffer size to retain in case of a retryable error during upload. Setting this to zero will disable upload buffering; this means that in the event of a failure during upload, the entire upload will be aborted and the client will likely receive an error. | +| `multipart_max_concurrent_uploads` | integer (uint) | — | 10 | Maximum number of concurrent `UploadPart` requests per `MultipartUpload`. | +| `insecure_allow_http` | boolean | — | false | Allow unencrypted HTTP connections. Only use this for local testing. | +| `disable_http2` | boolean | — | false | Disable HTTP/2 connections and only use HTTP/1.1. Default client configuration will have HTTP/1.1 and HTTP/2 enabled for connection schemes. HTTP/2 should be disabled if environments have poor support or performance related to HTTP/2. Safe to keep default unless underlying network environment, S3, or GCS API servers specify otherwise. | + +## CompressionAlgorithm + +Plus exactly one of the following variants (the key selects the variant): + +### `lz4` + +LZ4 compression algorithm is extremely fast for compression and +decompression, however does not perform very well in compression +ratio. In most cases build artifacts are highly compressible, however +lz4 is quite good at aborting early if the data is not deemed very +compressible. + +see: [https://lz4.github.io/lz4/](https://lz4.github.io/lz4/) + +**Type:** [Lz4Config](#lz4config) + +## StoreDirection + +| Value | Description | +| --- | --- | +| `"both"` | The store operates normally and all get and put operations are handled by it. | +| `"update"` | Update operations will cause persistence to this store, but Get operations will be ignored. This only makes sense on the fast store as the slow store will never get written to on Get anyway. | +| `"get"` | Get operations will cause persistence to this store, but Update operations will be ignored. | +| `"read_only"` | Operate as a read only store, only really makes sense if there's another way to write to it. | + +## ShardConfig + +Configuration for an individual shard of the store. + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `store` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | Store to shard the data to. | +| `weight` | integer (uint32) | — | 1 | The weight of the store. This is used to determine how much data should be sent to the store. The actual percentage is the sum of all the store's weights divided by the individual store's weight. | + +## GrpcEndpoint + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `address` | string | Yes | — | The endpoint address (i.e. grpc://example.com:443 or grpcs://example.com:443). | +| `tls_config` | [ClientTlsConfig](#clienttlsconfig) | — | — | The TLS configuration to use to connect to the endpoint (if grpcs). | +| `concurrency_limit` | integer (uint) | — | — | The maximum concurrency to allow on this endpoint. | +| `connect_timeout_s` | integer (uint64) | — | 30 seconds | Timeout for establishing a TCP connection to the endpoint (seconds). | +| `tcp_keepalive_s` | integer (uint64) | — | 30 seconds | TCP keepalive interval (seconds). Sends TCP keepalive probes at this interval to detect dead connections at the OS level. | +| `http2_keepalive_interval_s` | integer (uint64) | — | 30 seconds | HTTP/2 keepalive interval (seconds). Sends HTTP/2 PING frames at this interval to detect dead connections at the application level. | +| `http2_keepalive_timeout_s` | integer (uint64) | — | 20 seconds | HTTP/2 keepalive timeout (seconds). If a PING response is not received within this duration, the connection is considered dead. | + +## StoreType + +| Value | Description | +| --- | --- | +| `"cas"` | The store is content addressable storage. | +| `"ac"` | The store is an action cache. | + +## RedisMode + +| Value | Description | +| --- | --- | +| `"cluster"` | Use Redis Cluster. | +| `"sentinel"` | Use Redis Sentinel. | +| `"standard"` | Use a standalone Redis server. | + +## EndpointConfig + +Generic config for an endpoint and associated configs. + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `uri` | string | Yes | — | URI of the endpoint. | +| `timeout` | number (float) | — | 5 seconds | Timeout in seconds that a request should take. | +| `tls_config` | [ClientTlsConfig](#clienttlsconfig) | — | — | The TLS configuration to use to connect to the endpoint. | + +## UploadActionResultConfig + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `ac_store` | string | — | {No uploading is done} | Underlying AC store that the worker will use to publish execution results into. Objects placed in this store should be reachable from the scheduler/client-cas after they have finished updating. | +| `upload_ac_results_strategy` | [UploadCacheResultsStrategy](#uploadcacheresultsstrategy) | — | `SuccessOnly` | In which situations should the results be published to the `ac_store`, if set to `SuccessOnly` then only results with an exit code of 0 will be uploaded, if set to Everything all completed results will be uploaded. | +| `historical_results_store` | string | — | {CAS store of parent} | Store to upload historical results to. This should be a CAS store if set. | +| `upload_historical_results_strategy` | [UploadCacheResultsStrategy](#uploadcacheresultsstrategy) | — | `FailuresOnly` | In which situations should the results be published to the historical CAS. The historical CAS is where failures are published. These messages conform to the CAS key-value lookup format and are always a `HistoricalExecuteResponse` serialized message. | +| `success_message_template` | string | — | "" (no message) | Template to use for the `ExecuteResponse.message` property. This message is attached to the response before it is sent to the client. The following special variables are supported:; `digest_function`: Digest function used to calculate the action digest: `action_digest_hash`: Action digest hash: `action_digest_size`: Action digest size: `historical_results_hash`: `HistoricalExecuteResponse` digest hash: `historical_results_size`: `HistoricalExecuteResponse` digest size. | +| `failure_message_template` | string | — | "" (no message) | Same as `success_message_template` but for failure case. | + +## WorkerProperty + +Plus exactly one of the following variants (the key selects the variant): + +### `values` + +List of static values. +Note: Generally there should only ever be 1 value, but if the platform +property key is `PropertyType::Priority` it may have more than one value. + +**Type:** array of string + +### `query_cmd` + +A dynamic configuration. The string will be executed as a command +(not shell) and will be split by "\n" (new line character). + +**Type:** string + +## EnvironmentSource + +One of: + +- `property`: string — The name of the platform property in the action to get the value from. +- `value`: string — The raw value to set. +- `"from_environment"` — Take the value from the local environment corresponding to the name key +- `"timeout_millis"` — The max amount of time in milliseconds the command is allowed to run (requested by the client). +- `"side_channel_file"` — A special file path will be provided that can be used to communicate with the parent process about out-of-band information. This file will be read after the command has finished executing. Based on the contents of the file, the behavior of the result may be modified. +- `"action_directory"` — A "root" directory for the action. This directory can be used to store temporary files that are not needed after the action has completed. This directory will be purged after the action has completed. + +## DirectoryCacheConfig + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `max_entries` | integer (uint) | — | 1000 | Maximum number of cached directories. | +| `max_size_bytes` | integer (uint64) | — | 10737418240 (10 GB) | Maximum total size in bytes for all cached directories (0 = unlimited). | +| `cache_root` | string | — | `{work_directory}/../directory_cache` | Base directory for cache storage. This directory will be managed by the worker and should be on the same filesystem as `work_directory`. | + +## PropertyType + +When the scheduler matches tasks to workers that are capable of running +the task, this value will be used to determine how the property is treated. + +One of: + +- string +- `"minimum"` — Requires the platform property to be a u64 and when the scheduler looks for appropriate worker nodes that are capable of executing the task, the task will not run on a node that has less than this value. +- `"exact"` — Requires the platform property to be a string and when the scheduler looks for appropriate worker nodes that are capable of executing the task, the task will not run on a node that does not have this property set to the value with exact string match. +- `"priority"` — Does not restrict on this value and instead will be passed to the worker as an informational piece. In the future this will be used by the scheduler and worker to cause the scheduler to prefer certain workers over others, but not restrict them based on these values. + +## WorkerAllocationStrategy + +When a worker is being searched for to run a job, this will be used +on how to choose which worker should run the job when multiple +workers are able to run the task. + +| Value | Description | +| --- | --- | +| `"least_recently_used"` | Prefer workers that have been least recently used to run a job. | +| `"most_recently_used"` | Prefer workers that have been most recently used to run a job. | + +## ExperimentalSimpleSchedulerBackend + +One of: + +- `"memory"` — Use an in-memory store for the scheduler. +- `redis`: [ExperimentalRedisSchedulerBackend](#experimentalredisschedulerbackend) — Use a Redis store for the scheduler. + +## SchedulerSpec + +Plus exactly one of the following variants (the key selects the variant): + +### `simple` + +**Type:** [SimpleSpec](#simplespec) + +### `grpc` + +**Type:** [SchedulerGrpcSpec](#schedulergrpcspec) + +### `cache_lookup` + +**Type:** [CacheLookupSpec](#cachelookupspec) + +### `property_modifier` + +**Type:** [PropertyModifierSpec](#propertymodifierspec) + +### `historical_resource` + +**Type:** [HistoricalResourceSpec](#historicalresourcespec) + +## PropertyModification + +Plus exactly one of the following variants (the key selects the variant): + +### `add` + +Add a property to the action properties. + +**Type:** [PlatformPropertyAddition](#platformpropertyaddition) + +### `remove` + +Remove a named property from the action. + +**Type:** string + +### `replace` + +If a property is found, then replace it with another one. + +**Type:** [PlatformPropertyReplacement](#platformpropertyreplacement) + +## HttpListener + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `socket_address` | string | Yes | — | Address to listen on. Example: `127.0.0.1:8080` or `:8080` to listen to all IPs. | +| `freebind` | boolean | — | false | Allow binding `socket_address` before it is assigned locally. | +| `compression` | [HttpCompressionConfig](#httpcompressionconfig) | — | — | Data transport compression configuration to use for this service. | +| `advanced_http` | [HttpServerConfig](#httpserverconfig) | — | — | Advanced HTTP server configuration. | +| `max_decoding_message_size` | integer (uint) | — | 4 MiB | Maximum number of bytes to decode on each grpc stream chunk. | +| `tls` | [TlsConfig](#tlsconfig) | — | — | TLS configuration for this server. If not set, the server will not use TLS. | + +## CasServiceConfig + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `instance_name` | string | — | `""` | Used when the config references `instance_name` in the protocol. | +| `cas_store` | string | Yes | — | The store name referenced in the `stores` map in the main config. This store name referenced here may be reused multiple times. | +| `experimental_chunking` | [CasChunkingConfig](#caschunkingconfig) | — | not set — chunking RPCs are rejected, nothing is advertised, | Optional and experimental: enables the REAPI `SplitBlob`/`SpliceBlob` RPCs used by content-defined chunking clients (e.g. Bazel's `--experimental_remote_cache_chunking`). When set, the capabilities service advertises blob split/splice support and `FastCDC` 2020 parameters for this instance. When `cas_store` is a grpc store the RPCs are forwarded to the backend (which must support chunking with matching parameters); otherwise they are served locally. | + +## ActionCacheServiceConfig + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `instance_name` | string | — | `""` | Used when the config references `instance_name` in the protocol. | +| `ac_store` | string | Yes | — | The store name referenced in the `stores` map in the main config. This store name referenced here may be reused multiple times. | +| `read_only` | boolean | — | `false` | Whether the Action Cache store may be written to, this if set to false it is only possible to read from the Action Cache. | + +## CapabilitiesServiceConfig + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `instance_name` | string | — | `""` | Used when the config references `instance_name` in the protocol. | +| `remote_execution` | [CapabilitiesRemoteExecutionConfig](#capabilitiesremoteexecutionconfig) | — | — | Configuration for remote execution capabilities. If not set the capabilities service will inform the client that remote execution is not supported. | +| `remote_cache_compression` | boolean | — | — | Whether this instance supports Bazel remote cache compression. When enabled, the capabilities service advertises zstd wire compression and the ByteStream/CAS services accept REAPI compressed-blobs/zstd data. | + +## ExecutionServiceConfig + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `instance_name` | string | — | `""` | Used when the config references `instance_name` in the protocol. | +| `cas_store` | string | Yes | — | The store name referenced in the `stores` map in the main config. This store name referenced here may be reused multiple times. This value must be a CAS store reference. | +| `scheduler` | string | Yes | — | The scheduler name referenced in the `schedulers` map in the main config. | + +## ByteStreamServiceConfig + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `instance_name` | string | — | `""` | Used when the config references `instance_name` in the protocol. | +| `cas_store` | string | Yes | — | Name of the store in the "stores" configuration. | +| `max_bytes_per_stream` | integer (uint) | — | 64KiB | Max number of bytes to send on each grpc stream chunk. According to [https://github.com/grpc/grpc.github.io/issues/371](https://github.com/grpc/grpc.github.io/issues/371) 16KiB - 64KiB is optimal. | +| `persist_stream_on_disconnect_timeout_s` | integer (uint) | — | 10 seconds | In the event a client disconnects while uploading a blob, we will hold the internal stream open for this many seconds before closing it. This allows clients that disconnect to reconnect and continue uploading the same blob. | + +## FetchServiceConfig + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `instance_name` | string | — | `""` | Used when the config references `instance_name` in the protocol. | +| `fetch_store` | string | Yes | — | The store name referenced in the `stores` map in the main config. This store name referenced here may be reused multiple times. | + +## PushServiceConfig + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `instance_name` | string | — | `""` | Used when the config references `instance_name` in the protocol. | +| `push_store` | string | Yes | — | The store name referenced in the `stores` map in the main config. This store name referenced here may be reused multiple times. | +| `read_only` | boolean | — | `false` | Whether the Action Cache store may be written to, this if set to false it is only possible to read from the Action Cache. | + +## WorkerApiConfig + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `scheduler` | string | Yes | — | The scheduler name referenced in the `schedulers` map in the main config. | + +## BepConfig + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `store` | string | Yes | — | The store to publish build events to. The store name referenced in the `stores` map in the main config. | + +## AdminConfig + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `path` | string | — | "/admin" | Path to register the admin API. If path is "/admin", and your domain is "example.com", you can reach the endpoint with: [http://example.com/admin](http://example.com/admin). | + +## HealthConfig + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `path` | string | — | "/status" | Path to register the health status check. If path is "/status", and your domain is "example.com", you can reach the endpoint with: [http://example.com/status](http://example.com/status). | +| `timeout_seconds` | integer (uint64) | — | 5s | Timeout on health checks. | + +## ErrorCode + +The possible error codes that might occur on an upstream request. + +| Value | Description | +| --- | --- | +| `"Cancelled"` | | +| `"Unknown"` | | +| `"InvalidArgument"` | | +| `"DeadlineExceeded"` | | +| `"NotFound"` | | +| `"AlreadyExists"` | | +| `"PermissionDenied"` | | +| `"ResourceExhausted"` | | +| `"FailedPrecondition"` | | +| `"Aborted"` | | +| `"OutOfRange"` | | +| `"Unimplemented"` | | +| `"Internal"` | | +| `"Unavailable"` | | +| `"DataLoss"` | | +| `"Unauthenticated"` | | + +## Lz4Config + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `block_size` | integer (uint32) | — | 65536 (64k) | Size of the blocks to compress. Higher values require more ram, but might yield slightly better compression ratios. | +| `max_decode_block_size` | integer (uint32) | — | value in `block_size` | Maximum size allowed to attempt to deserialize data into. This is needed because the `block_size` is embedded into the data so if there was a bad actor, they could upload an extremely large `block_size`'ed entry and we'd allocate a large amount of memory when retrieving the data. To prevent this from happening, we allow you to specify the maximum that we'll attempt to deserialize. | + +## ClientTlsConfig + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `ca_file` | string | — | — | Path to the certificate authority to use to validate the remote. | +| `cert_file` | string | — | — | Path to the certificate file for client authentication. | +| `key_file` | string | — | — | Path to the private key file for client authentication. | +| `use_native_roots` | boolean | — | false | If set the client will use the native roots for TLS connections. | + +## UploadCacheResultsStrategy + +| Value | Description | +| --- | --- | +| `"success_only"` | Only upload action results with an exit code of 0. | +| `"never"` | Don't upload any action results. | +| `"everything"` | Upload all action results that complete. | +| `"failures_only"` | Only upload action results that fail. | + +## ExperimentalRedisSchedulerBackend + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `redis_store` | string | Yes | — | A reference to the Redis store to use for the scheduler. Note: This MUST resolve to a `RedisSpec`. | + +## PlatformPropertyAddition + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `name` | string | Yes | — | The name of the property to add. | +| `value` | string | Yes | — | The value to assign to the property. | + +## PlatformPropertyReplacement + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `name` | string | Yes | — | The name of the property to replace. | +| `value` | string | — | — | The value to match against, if unset then any instance matches. | +| `new_name` | string | Yes | — | The new name of the property. | +| `new_value` | string | — | — | The value to assign to the property, if unset will remain the same. | + +## HttpCompressionConfig + +Note: Compressing data in the cloud rarely has a benefit, since most +cloud providers have very high bandwidth backplanes. However, for +clients not inside the data center, it might be a good idea to +compress data to and from the cloud. This will however come at a high +CPU and performance cost. If you are making remote execution share the +same CAS/AC servers as client's remote cache, you can create multiple +services with different compression settings that are served on +different ports. Then configure the non-cloud clients to use one port +and cloud-clients to use another. + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `send_compression_algorithm` | [HttpCompressionAlgorithm](#httpcompressionalgorithm) | — | `HttpCompressionAlgorithm::None` | The compression algorithm that the server will use when sending responses to clients. Enabling this will likely save a lot of data transfer, but will consume a lot of CPU and add a lot of latency. see: [https://github.com/tracemachina/nativelink/issues/109](https://github.com/tracemachina/nativelink/issues/109) | +| `accepted_compression_algorithms` | array of [HttpCompressionAlgorithm](#httpcompressionalgorithm) | Yes | {no supported compression} | The compression algorithm that the server will accept from clients. The server will broadcast the supported compression algorithms to clients and the client will choose which compression algorithm to use. Enabling this will likely save a lot of data transfer, but will consume a lot of CPU and add a lot of latency. see: [https://github.com/tracemachina/nativelink/issues/109](https://github.com/tracemachina/nativelink/issues/109) | + +## HttpServerConfig + +Advanced HTTP configuration. These generally should not be set. +For documentation on these settings, see the hyper documentation: +See: [hyper HTTP server docs](https://docs.rs/hyper/latest/hyper/server/conn/struct.Http.html) + +Note: All of these default to the default values from hyper unless otherwise +specified. + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `http2_keep_alive_interval` | integer (uint32) | — | — | Interval to send keep-alive pings via HTTP2. Note: This is in seconds. | +| `experimental_http2_max_pending_accept_reset_streams` | integer (uint32) | — | — | | +| `experimental_http2_initial_stream_window_size` | integer (uint32) | — | — | | +| `experimental_http2_initial_connection_window_size` | integer (uint32) | — | — | | +| `experimental_http2_adaptive_window` | boolean | — | — | | +| `experimental_http2_max_frame_size` | integer (uint32) | — | — | | +| `experimental_http2_max_concurrent_streams` | integer (uint32) | — | — | | +| `experimental_http2_keep_alive_timeout_s` | integer (uint32) | — | — | | +| `experimental_http2_max_send_buf_size` | integer (uint32) | — | — | | +| `experimental_http2_enable_connect_protocol` | boolean | — | — | | +| `experimental_http2_max_header_list_size` | integer (uint32) | — | — | | + +## TlsConfig + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `cert_file` | string | Yes | — | Path to the certificate file. | +| `key_file` | string | Yes | — | Path to the private key file. | +| `client_ca_file` | string | — | — | Path to the certificate authority for mTLS, if client authentication is required for this endpoint. | +| `client_crl_file` | string | — | — | Path to the certificate revocation list for mTLS, if client authentication is required for this endpoint. | + +## CasChunkingConfig + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `index_store` | string | — | — | The store name referenced in the `stores` map in the main config used to persist blob-to-chunks layouts. Keys are the digests of the original blobs and values are serialized chunk layouts (which do not hash to those digests), so this store MUST NOT perform content digest verification and MUST NOT be the same store as `cas_store` — writing layouts into the CAS would overwrite blob content. Using the same store name as `cas_store` is rejected at startup. | +| `avg_chunk_size_bytes` | integer (uint64) | — | 524288 (512 KiB) | The average chunk size in bytes advertised to clients through the `FastCDC` 2020 capability parameters and used for server-side chunking in `SplitBlob`. Clients derive the minimum and maximum chunk sizes from this value (avg / 4 and avg * 4). The value must be between 1 KiB and 1 MiB. | +| `max_chunk_count` | integer (uint64) | — | 50000 | Maximum number of chunks accepted in a `SpliceBlob` request or produced by on-demand chunking in `SplitBlob`. Blobs that would produce more chunks are served without chunking (`SplitBlob` returns `NOT_FOUND` and clients fall back to a regular download). This bounds the size of stored chunk layouts and of `SplitBlobResponse` messages (roughly 80-140 bytes per chunk). At the default average chunk size the default cap supports blobs up to ~25 GiB; note that values above ~50000 may produce responses that exceed default gRPC message size limits on clients. | + +## CapabilitiesRemoteExecutionConfig + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `scheduler` | string | Yes | — | Scheduler used to configure the capabilities of remote execution. | + +## HttpCompressionAlgorithm + +| Value | Description | +| --- | --- | +| `"none"` | No compression. | +| `"gzip"` | Zlib compression. | + +## Reading the source + +If anything here disagrees with the binary, the source wins: + +- [`stores.rs`](https://github.com/TraceMachina/nativelink/tree/v1.6.1/nativelink-config/src/stores.rs) +- [`cas_server.rs`](https://github.com/TraceMachina/nativelink/tree/v1.6.1/nativelink-config/src/cas_server.rs) +- [`schedulers.rs`](https://github.com/TraceMachina/nativelink/tree/v1.6.1/nativelink-config/src/schedulers.rs) diff --git a/web/apps/docs/lib/config-versions.ts b/web/apps/docs/lib/config-versions.ts index 5ba51fd01..2c7b7172f 100644 --- a/web/apps/docs/lib/config-versions.ts +++ b/web/apps/docs/lib/config-versions.ts @@ -28,13 +28,21 @@ export const CONFIG_VERSIONS: ConfigVersion[] = [ "isDev": true }, { - "version": "v1.6.1", - "label": "v1.6.1 (latest)", + "version": "v1.6.2", + "label": "v1.6.2 (latest)", "href": "/reference/nativelink-config", - "ref": "v1.6.1", + "ref": "v1.6.2", "isLatest": true, "isDev": false }, + { + "version": "v1.6.1", + "label": "v1.6.1", + "href": "/reference/nativelink-config/v1.6.1", + "ref": "v1.6.1", + "isLatest": false, + "isDev": false + }, { "version": "v1.6.0", "label": "v1.6.0", From 5ee0afcfbff2a2393ea3a31b5298e0f38ead1448 Mon Sep 17 00:00:00 2001 From: Tom Parker-Shemilt Date: Sat, 18 Jul 2026 17:29:39 +0100 Subject: [PATCH 20/84] Cache cargo runs under nix (#2570) Co-authored-by: Marcus Eagan --- .github/workflows/native-cargo.yaml | 4 ++-- .github/workflows/nix.yaml | 4 ++++ 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/workflows/native-cargo.yaml b/.github/workflows/native-cargo.yaml index 2b6789598..1bc1a4523 100644 --- a/.github/workflows/native-cargo.yaml +++ b/.github/workflows/native-cargo.yaml @@ -47,8 +47,8 @@ jobs: run: rustup update && rustup default ${{ matrix.toolchain }} - name: Rust cache - # https://github.com/Swatinem/rust-cache/releases/tag/v2.9.1 - uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 + uses: >- # v2.9.1 + Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 - name: Build on ${{ runner.os }} run: cargo build --all --profile=smol diff --git a/.github/workflows/nix.yaml b/.github/workflows/nix.yaml index 009061aec..cc12eb81a 100644 --- a/.github/workflows/nix.yaml +++ b/.github/workflows/nix.yaml @@ -90,6 +90,10 @@ jobs: with: nativelink_attic_token: ${{ secrets.NATIVELINK_ATTIC_TOKEN }} + - name: Rust cache + uses: >- # v2.9.1 + Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 + - name: Invoke Cargo build in Nix shell run: > nix develop --impure --command From eed5b31e16d2853c2dffddfa107324cd0d7ba0a7 Mon Sep 17 00:00:00 2001 From: Marcus Eagan Date: Mon, 20 Jul 2026 01:01:52 +0100 Subject: [PATCH 21/84] Bootstrap CI on NativeLink Cloud staging (remote cache + BES, dark) (#2579) --- .bazelrc | 3 + .../setup-nativelink-cloud/action.yaml | 131 ++++++++++++++++++ .github/workflows/lre.yaml | 12 ++ .github/workflows/native-bazel.yaml | 24 ++++ .../workflows/nativelink-cloud-canary.yaml | 104 ++++++++++++++ .github/workflows/nix.yaml | 12 ++ .github/workflows/sanitizers.yaml | 12 ++ ci.bazelrc | 54 ++++++++ 8 files changed, 352 insertions(+) create mode 100644 .github/actions/setup-nativelink-cloud/action.yaml create mode 100644 .github/workflows/nativelink-cloud-canary.yaml create mode 100644 ci.bazelrc diff --git a/.bazelrc b/.bazelrc index 6fdc8a0c1..704b90ea9 100644 --- a/.bazelrc +++ b/.bazelrc @@ -232,5 +232,8 @@ try-import %workspace%/nixos.bazelrc # Generated by the nativelink flake module. try-import %workspace%/nativelink.bazelrc +# Opt-in NativeLink Cloud configs used by this repo's CI. +try-import %workspace%/ci.bazelrc + # Allow user-side customization. try-import %workspace%/user.bazelrc diff --git a/.github/actions/setup-nativelink-cloud/action.yaml b/.github/actions/setup-nativelink-cloud/action.yaml new file mode 100644 index 000000000..fc34e6b90 --- /dev/null +++ b/.github/actions/setup-nativelink-cloud/action.yaml @@ -0,0 +1,131 @@ +--- +name: Setup NativeLink Cloud +description: >- + Writes a user.bazelrc that opts this job into NativeLink Cloud remote + cache/BES (and optionally remote execution) via the configs in ci.bazelrc. + Endpoints are derived from the claim-base input, which comes from a + repository secret, so no claim hostname lives in the repository. No-ops + when the API key or claim base is unavailable (fork PRs have no secrets) + or when ci-mode is 'off', so builds degrade to plain local execution. + +inputs: + api-key: + description: >- + NativeLink Cloud API key. Empty (fork PRs have no secrets) disables the + setup entirely. + required: false + default: '' + claim-base: + description: >- + Claim endpoint base as ., normally passed from + secrets.NATIVELINK_STAGING_CLAIM_BASE. The action derives the + grpcs://cas-, bes-, and scheduler- endpoints from it. Empty disables + the setup. + required: false + default: '' + bes-results-url: + description: >- + Optional console URL prefix for invocation links, normally passed from + secrets.NATIVELINK_STAGING_BES_RESULTS_URL. Omitted from the build when + empty. + required: false + default: '' + ci-mode: + description: >- + Kill switch, normally passed as vars.NATIVELINK_CI_MODE. Empty or 'off' + disables; 'cache' enables remote cache + BES; 'rbe' additionally honors + exec: on. The canary workflow pins this to 'cache'/'rbe' so it stays + live while regular lanes are dark. + required: false + default: '' + mode: + description: >- + 'read' (PR lanes; never uploads action results) or 'write' (main-branch + push lanes; only TraceMachina main-branch identities hold write keys). + required: false + default: read + exec: + description: >- + 'on' requests remote execution. Honored only when ci-mode is 'rbe', and + only on Linux runners. + required: false + default: 'off' + container-image: + description: >- + Exec property attached when remote execution is honored. Must + byte-for-byte match the container-image the claim's worker pool + advertises, or actions queue forever. + required: false + default: debian:bookworm-slim + +outputs: + enabled: + description: "'true' when a user.bazelrc was written, else 'false'." + value: ${{ steps.configure.outputs.enabled }} + +runs: + using: composite + steps: + - name: Write user.bazelrc + id: configure + shell: bash + env: + NL_API_KEY: ${{ inputs.api-key }} + NL_CLAIM_BASE: ${{ inputs.claim-base }} + NL_BES_RESULTS_URL: ${{ inputs.bes-results-url }} + NL_CI_MODE: ${{ inputs.ci-mode }} + NL_MODE: ${{ inputs.mode }} + NL_EXEC: ${{ inputs.exec }} + NL_CONTAINER_IMAGE: ${{ inputs.container-image }} + run: | + set -euo pipefail + + if [[ -z "$NL_API_KEY" || -z "$NL_CLAIM_BASE" || -z "$NL_CI_MODE" || "$NL_CI_MODE" == "off" ]]; then + echo "enabled=false" >> "$GITHUB_OUTPUT" + echo "NativeLink Cloud disabled (no API key/claim or ci-mode off); building locally." + exit 0 + fi + + exec_enabled=false + if [[ "$NL_EXEC" == "on" ]]; then + if [[ "$RUNNER_OS" != "Linux" ]]; then + # macOS auto-applies LRE platforms from lre.bazelrc that conflict + # with the nl-rbe Linux platform, and no non-Linux workers exist. + echo "exec: on is only supported on Linux runners (got $RUNNER_OS)" >&2 + exit 1 + fi + if [[ "$NL_CI_MODE" == "rbe" ]]; then + exec_enabled=true + fi + fi + + { + echo "common --config=nl-cache" + echo "common --remote_cache=grpcs://cas-${NL_CLAIM_BASE}" + echo "common --bes_backend=grpcs://bes-${NL_CLAIM_BASE}" + if [[ -n "$NL_BES_RESULTS_URL" ]]; then + echo "common --bes_results_url=${NL_BES_RESULTS_URL}" + fi + if [[ "$exec_enabled" == true ]]; then + echo "common --remote_executor=grpcs://scheduler-${NL_CLAIM_BASE}:443" + echo "common --remote_default_exec_properties=container-image=${NL_CONTAINER_IMAGE}" + echo "common --config=nl-rbe" + fi + if [[ "$NL_MODE" == "write" ]]; then + echo "common --config=nl-write" + else + # Plain (non-config) line so it also beats the nix-generated + # nativelink.bazelrc on lanes that run under nix develop. + echo "common --remote_upload_local_results=false" + fi + # Both headers from the same variable: a truncated BES key once + # caused silent 401s while the cache header kept working. + echo "common --remote_header=x-nativelink-api-key=${NL_API_KEY}" + echo "common --bes_header=x-nativelink-api-key=${NL_API_KEY}" + echo "common --remote_header=x-nativelink-project=nativelink-github-ci" + echo "common --build_metadata=ROLE=CI" + } >> user.bazelrc + + echo "enabled=true" >> "$GITHUB_OUTPUT" + # Deliberately do not echo the endpoints: the claim base is a secret. + echo "NativeLink Cloud enabled: mode=${NL_MODE} exec=${exec_enabled}" diff --git a/.github/workflows/lre.yaml b/.github/workflows/lre.yaml index 61927eca8..a9229cc12 100644 --- a/.github/workflows/lre.yaml +++ b/.github/workflows/lre.yaml @@ -42,6 +42,18 @@ jobs: with: nativelink_attic_token: ${{ secrets.NATIVELINK_ATTIC_TOKEN }} + - name: Setup NativeLink Cloud + uses: ./.github/actions/setup-nativelink-cloud + with: + ci-mode: ${{ vars.NATIVELINK_CI_MODE }} + claim-base: ${{ secrets.NATIVELINK_STAGING_CLAIM_BASE }} + bes-results-url: ${{ secrets.NATIVELINK_STAGING_BES_RESULTS_URL }} + api-key: >- + ${{ github.event_name == 'push' + && secrets.NATIVELINK_STAGING_API_KEY_RW + || secrets.NATIVELINK_STAGING_API_KEY_RO }} + mode: ${{ github.event_name == 'push' && 'write' || 'read' }} + - name: Build example with ${{ matrix.toolchain }} toolchain env: TOOLCHAIN: ${{ matrix.toolchain }} diff --git a/.github/workflows/native-bazel.yaml b/.github/workflows/native-bazel.yaml index b38d65a74..7161dab46 100644 --- a/.github/workflows/native-bazel.yaml +++ b/.github/workflows/native-bazel.yaml @@ -60,6 +60,18 @@ jobs: repository-cache: true disk-cache: ${{ github.workflow }}-${{ matrix.os }}-bazel-${{ matrix.bazel_version }} + - name: Setup NativeLink Cloud + uses: ./.github/actions/setup-nativelink-cloud + with: + ci-mode: ${{ vars.NATIVELINK_CI_MODE }} + claim-base: ${{ secrets.NATIVELINK_STAGING_CLAIM_BASE }} + bes-results-url: ${{ secrets.NATIVELINK_STAGING_BES_RESULTS_URL }} + api-key: >- + ${{ github.event_name == 'push' + && secrets.NATIVELINK_STAGING_API_KEY_RW + || secrets.NATIVELINK_STAGING_API_KEY_RO }} + mode: ${{ github.event_name == 'push' && 'write' || 'read' }} + - name: Show Bazel version run: bazel --version @@ -115,6 +127,18 @@ jobs: repository-cache: true disk-cache: ${{ github.workflow }}-ubuntu-24.04 + - name: Setup NativeLink Cloud + uses: ./.github/actions/setup-nativelink-cloud + with: + ci-mode: ${{ vars.NATIVELINK_CI_MODE }} + claim-base: ${{ secrets.NATIVELINK_STAGING_CLAIM_BASE }} + bes-results-url: ${{ secrets.NATIVELINK_STAGING_BES_RESULTS_URL }} + api-key: >- + ${{ github.event_name == 'push' + && secrets.NATIVELINK_STAGING_API_KEY_RW + || secrets.NATIVELINK_STAGING_API_KEY_RO }} + mode: ${{ github.event_name == 'push' && 'write' || 'read' }} + - name: Run Store tester with sentinel run: | bazel run //:redis_store_tester \ diff --git a/.github/workflows/nativelink-cloud-canary.yaml b/.github/workflows/nativelink-cloud-canary.yaml new file mode 100644 index 000000000..60763c4ef --- /dev/null +++ b/.github/workflows/nativelink-cloud-canary.yaml @@ -0,0 +1,104 @@ +--- +name: NativeLink Cloud Canary + +on: + schedule: + - cron: '17 6 * * *' + workflow_dispatch: + pull_request: + branches: [main] + paths: + - ci.bazelrc + - .github/actions/setup-nativelink-cloud/** + - .github/workflows/nativelink-cloud-canary.yaml + +permissions: read-all + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} + +jobs: + cache-canary: + name: Remote cache canary + runs-on: ubuntu-24.04 + # Also the queue-forever tripwire once an RBE leg is added. + timeout-minutes: 15 + steps: + - name: Checkout + uses: >- # v6.0.2 + actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd + + - name: Setup Bazel + uses: >- # v0.19.0 + bazel-contrib/setup-bazel@c5acdfb288317d0b5c0bbd7a396a3dc868bb0f86 + with: + bazelisk-cache: true + repository-cache: true + # No disk-cache: remote hits must be the only cache signal here. + + - name: Setup NativeLink Cloud + id: nl + uses: ./.github/actions/setup-nativelink-cloud + with: + # The canary ignores vars.NATIVELINK_CI_MODE on purpose: it must + # prove the cloud path is healthy while regular lanes are still + # dark. With no secrets configured (or on fork PRs) it no-ops. + ci-mode: cache + claim-base: ${{ secrets.NATIVELINK_STAGING_CLAIM_BASE }} + bes-results-url: ${{ secrets.NATIVELINK_STAGING_BES_RESULTS_URL }} + api-key: >- + ${{ github.event_name == 'pull_request' + && secrets.NATIVELINK_STAGING_API_KEY_RO + || secrets.NATIVELINK_STAGING_API_KEY_RW }} + mode: ${{ github.event_name == 'pull_request' && 'read' || 'write' }} + + - name: Build (populate cache) + if: steps.nl.outputs.enabled == 'true' + run: | + bazel build //nativelink-config/... \ + --lockfile_mode=error \ + --verbose_failures \ + --bes_upload_mode=wait_for_upload_complete + + # The hit-rate assertion needs the populate step to have uploaded, so it + # only runs with the write key (schedule/dispatch/push — never PRs). + - name: Drop local build state + if: steps.nl.outputs.enabled == 'true' && github.event_name != 'pull_request' + run: | + bazel clean + bazel shutdown + + - name: Rebuild and assert remote cache hits + if: steps.nl.outputs.enabled == 'true' && github.event_name != 'pull_request' + run: | + bazel build //nativelink-config/... \ + --lockfile_mode=error \ + --verbose_failures \ + --bes_upload_mode=wait_for_upload_complete \ + --build_event_json_file="$RUNNER_TEMP/bep.json" + + hits=$(jq -s '[.[] | .buildMetrics.actionSummary.runnerCount[]? + | select(.name == "remote cache hit") | .count] | add // 0' \ + "$RUNNER_TEMP/bep.json") + total=$(jq -s '[.[] | .buildMetrics.actionSummary.runnerCount[]? + | select(.name != "internal") | .count] | add // 0' \ + "$RUNNER_TEMP/bep.json") + invocation=$(jq -rs '[.[] | select(.id.started != null) + | .started.uuid][0] // empty' "$RUNNER_TEMP/bep.json") + + echo "Remote cache hits on rebuild: ${hits}/${total} (invocation ${invocation})" + { + echo "### NativeLink Cloud canary" + echo "- Remote cache hits on rebuild: **${hits}/${total}**" + echo "- Invocation: \`${invocation}\`" + } >> "$GITHUB_STEP_SUMMARY" + + if [[ "$total" -eq 0 ]]; then + echo "FAIL: no actions were measured; the assertion is vacuous" >&2 + exit 1 + fi + if (( hits * 100 < total * 90 )); then + echo "FAIL: remote cache hit rate below 90% on an unchanged rebuild" >&2 + exit 1 + fi diff --git a/.github/workflows/nix.yaml b/.github/workflows/nix.yaml index cc12eb81a..31a7fae90 100644 --- a/.github/workflows/nix.yaml +++ b/.github/workflows/nix.yaml @@ -38,6 +38,18 @@ jobs: with: nativelink_attic_token: ${{ secrets.NATIVELINK_ATTIC_TOKEN }} + - name: Setup NativeLink Cloud + uses: ./.github/actions/setup-nativelink-cloud + with: + ci-mode: ${{ vars.NATIVELINK_CI_MODE }} + claim-base: ${{ secrets.NATIVELINK_STAGING_CLAIM_BASE }} + bes-results-url: ${{ secrets.NATIVELINK_STAGING_BES_RESULTS_URL }} + api-key: >- + ${{ github.event_name == 'push' + && secrets.NATIVELINK_STAGING_API_KEY_RW + || secrets.NATIVELINK_STAGING_API_KEY_RO }} + mode: ${{ github.event_name == 'push' && 'write' || 'read' }} + - name: Invoke Bazel build in Nix shell run: | set -o pipefail diff --git a/.github/workflows/sanitizers.yaml b/.github/workflows/sanitizers.yaml index 0a867f368..2eb7aa65c 100644 --- a/.github/workflows/sanitizers.yaml +++ b/.github/workflows/sanitizers.yaml @@ -44,5 +44,17 @@ jobs: bazelisk-cache: true repository-cache: true + - name: Setup NativeLink Cloud + uses: ./.github/actions/setup-nativelink-cloud + with: + ci-mode: ${{ vars.NATIVELINK_CI_MODE }} + claim-base: ${{ secrets.NATIVELINK_STAGING_CLAIM_BASE }} + bes-results-url: ${{ secrets.NATIVELINK_STAGING_BES_RESULTS_URL }} + api-key: >- + ${{ github.event_name == 'push' + && secrets.NATIVELINK_STAGING_API_KEY_RW + || secrets.NATIVELINK_STAGING_API_KEY_RO }} + mode: ${{ github.event_name == 'push' && 'write' || 'read' }} + - name: Run Bazel tests run: bazel test --config=${{ matrix.sanitizer }} --lockfile_mode=error --verbose_failures //... diff --git a/ci.bazelrc b/ci.bazelrc new file mode 100644 index 000000000..90a314957 --- /dev/null +++ b/ci.bazelrc @@ -0,0 +1,54 @@ +# Copyright 2022 The NativeLink Authors. All rights reserved. +# +# Licensed under the Functional Source License, Version 1.1, Apache 2.0 Future License (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# See LICENSE file for details +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# NativeLink Cloud configuration for this repository's own CI. Everything here +# is opt-in via --config; plain local builds are unaffected. CI activates these +# groups through a generated user.bazelrc (see +# .github/actions/setup-nativelink-cloud), which injects the claim endpoints +# and the --remote_header/--bes_header credentials from repository secrets — +# neither endpoints nor secrets belong in this file. + +# --- Behavior groups (endpoint-free) --- + +# Remote cache + BES for lanes that still execute locally. Safe on any host. +common:nl-cache --remote_timeout=600 +common:nl-cache --remote_retries=5 +common:nl-cache --build_event_publish_all_actions +common:nl-cache --bes_upload_mode=fully_async + +# Main-branch mode: populate the shared cache. Only TraceMachina main-branch +# identities may hold a write-capable key; PR lanes instead get a plain +# --remote_upload_local_results=false line from the setup action. +common:nl-write --remote_upload_local_results=true +common:nl-write --remote_cache_async + +# Remote execution. Mirrors the staging-rbe config proven from a developer +# machine: hermetic LLVM plus downloaded Rust toolchains, so workers need no +# GCC/Rust/Nix. Combine only with an *-exec endpoint group, and only on +# x86_64 Linux hosts — macOS auto-applies conflicting LRE platforms from +# lre.bazelrc, which the setup action guards against. +common:nl-rbe --platforms=@llvm//platforms:linux_x86_64_gnu.2.28 +common:nl-rbe --host_platform=@llvm//platforms:linux_x86_64_gnu.2.28 +common:nl-rbe --extra_execution_platforms=@llvm//platforms:linux_x86_64_gnu.2.28 +common:nl-rbe --extra_toolchains=@rust_toolchains//:all +# The default musl interpreter is dynamically linked against the musl loader, +# which glibc-only worker images lack (genrule tools exit 127). +common:nl-rbe --@rules_python//python/config_settings:py_linux_libc=glibc +common:nl-rbe --jobs=200 +common:nl-rbe --remote_download_minimal +common:nl-rbe --remote_retries=8 +common:nl-rbe --experimental_remote_cache_eviction_retries=5 + +# Endpoints deliberately do not appear in this file: the setup action derives +# them from the claim-base repository secret at job time. From abd025888d9b553380ac169f73f802005987749b Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Mon, 20 Jul 2026 09:44:26 +0100 Subject: [PATCH 22/84] Update Rust crate syn to v3 (#2578) Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> Co-authored-by: Marcus Eagan --- Cargo.lock | 121 ++++++++++-------- MODULE.bazel.lock | 1 + nativelink-macro/Cargo.toml | 7 +- .../nativelink-metric-macro-derive/Cargo.toml | 7 +- .../nativelink-metric-macro-derive/src/lib.rs | 3 - nativelink-test/fuzz/Cargo.lock | 69 +++++----- 6 files changed, 119 insertions(+), 89 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 2eea85292..bcef4a231 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -165,7 +165,7 @@ checksum = "c7c24de15d275a1ecfd47a380fb4d5ec9bfe0933f309ed5e705b775596a3574d" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -176,7 +176,7 @@ checksum = "9035ad2d096bed7955a320ee7e2230574d28fd3c3a0f186cbea1ff3c7eed5dbb" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -586,7 +586,7 @@ checksum = "8d7396fd9500589e62e460e987ecb671bad374934e55ec3b5f498cc7a8a8a7b7" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -721,7 +721,7 @@ checksum = "b9b52dba6a345f3ad2d42ff8d0d63df9d0994cfa29657bf18ffdbf149f78a4f5" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", "tracing", ] @@ -1053,7 +1053,7 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -1323,7 +1323,7 @@ checksum = "f46882e17999c6cc590af592290432be3bce0428cb0d5f8b6715e4dc7b383eb3" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -1346,7 +1346,7 @@ dependencies = [ "proc-macro2", "quote", "strsim", - "syn", + "syn 2.0.117", ] [[package]] @@ -1357,7 +1357,7 @@ checksum = "ac3984ec7bd6cfa798e62b4a642426a5be0e68f9401cfc2a01e3fa9ea2fcdb8d" dependencies = [ "darling_core", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -1395,7 +1395,7 @@ checksum = "d65d7ce8132b7c0e54497a4d9a55a1c2a0912a0d786cf894472ba818fba45762" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -1406,7 +1406,7 @@ checksum = "ef941ded77d15ca19b40374869ac6000af1c9f2a4c0f3d4c70926287e6364a8f" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -1419,7 +1419,7 @@ dependencies = [ "proc-macro2", "quote", "rustc_version", - "syn", + "syn 2.0.117", ] [[package]] @@ -1440,7 +1440,7 @@ dependencies = [ "proc-macro2", "quote", "rustc_version", - "syn", + "syn 2.0.117", "unicode-xid", ] @@ -1491,7 +1491,7 @@ checksum = "1ac70aa55017e108007fbaf5aa0f54b021c98f92ff8af59d42eda9da96e3dd4f" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -1791,7 +1791,7 @@ checksum = "e835b70203e41293343137df5c0664546da5745f82ec9b84d40be8336958447b" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -2282,7 +2282,7 @@ dependencies = [ "libc", "percent-encoding", "pin-project-lite", - "socket2 0.5.10", + "socket2 0.6.4", "tokio", "tower-service", "tracing", @@ -2542,7 +2542,7 @@ dependencies = [ "quote", "rustc_version", "simd_cesu8", - "syn", + "syn 2.0.117", ] [[package]] @@ -2567,7 +2567,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "38c0b942f458fe50cdac086d2f946512305e5631e720728f2a61aabcd47a6264" dependencies = [ "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -2713,7 +2713,7 @@ dependencies = [ "macro_magic_core", "macro_magic_macros", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -2727,7 +2727,7 @@ dependencies = [ "macro_magic_core_macros", "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -2738,7 +2738,7 @@ checksum = "b02abfe41815b5bd98dbd4260173db2c116dda171dc0fe7838cb206333b83308" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -2749,7 +2749,7 @@ checksum = "73ea28ee64b88876bf45277ed9a5817c1817df061a74f2b988971a12570e5869" dependencies = [ "macro_magic_core", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -2952,7 +2952,7 @@ dependencies = [ "macro_magic", "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -3041,7 +3041,7 @@ version = "1.6.2" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 3.0.1", ] [[package]] @@ -3061,7 +3061,7 @@ version = "1.6.2" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 3.0.1", ] [[package]] @@ -3713,7 +3713,7 @@ dependencies = [ "pest_meta", "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -3754,7 +3754,7 @@ checksum = "6e918e4ff8c4549eb882f14b3a4bc8c8bc93de829416eacf579f1207a8fbf861" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -3854,7 +3854,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" dependencies = [ "proc-macro2", - "syn", + "syn 2.0.117", ] [[package]] @@ -3900,7 +3900,7 @@ dependencies = [ "prost", "prost-types", "regex", - "syn", + "syn 2.0.117", "tempfile", ] @@ -3914,7 +3914,7 @@ dependencies = [ "itertools", "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -4133,7 +4133,7 @@ checksum = "b7186006dcb21920990093f30e3dea63b7d6e977bf1256be20c3563a5db070da" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -4487,7 +4487,7 @@ dependencies = [ "proc-macro2", "quote", "serde_derive_internals", - "syn", + "syn 2.0.117", ] [[package]] @@ -4582,7 +4582,7 @@ checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -4593,7 +4593,7 @@ checksum = "18d26a20a969b9e3fdf2fc2d9f21eda6c40e2de84c9408bb5d3b05d499aae711" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -4671,7 +4671,7 @@ dependencies = [ "darling", "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -4695,7 +4695,7 @@ checksum = "94e153fc76e1c6a068703d6d29c508a0b15c061c4b7e43da59cc097bc342673c" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -4901,6 +4901,17 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "syn" +version = "3.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5edbec4ed188954a10c12c038215f8ce7606b2d5c973cd8dc43e8795065c5f2f" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + [[package]] name = "sync_wrapper" version = "1.0.2" @@ -4918,7 +4929,7 @@ checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -5009,7 +5020,7 @@ checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -5020,7 +5031,7 @@ checksum = "ebc4ee7f67670e9b64d05fa4253e753e016c6c95ff35b89b7941d6b856dec1d5" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -5131,7 +5142,7 @@ checksum = "385a6cb71ab9ab790c5fe8d67f1645e6c450a7ce006a33de03daa956cf70a496" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -5211,7 +5222,7 @@ dependencies = [ "prettyplease", "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -5236,7 +5247,7 @@ dependencies = [ "prost-build", "prost-types", "quote", - "syn", + "syn 2.0.117", "tempfile", "tonic-build", ] @@ -5320,7 +5331,7 @@ checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -5408,7 +5419,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "04659ddb06c87d233c566112c1c9c5b9e98256d9af50ec3bc9c8327f873a7568" dependencies = [ "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -5434,7 +5445,7 @@ checksum = "3c36781cc0e46a83726d9879608e4cf6c2505237e263a8eb8c24502989cfdb28" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -5498,7 +5509,7 @@ dependencies = [ "proc-macro2", "quote", "rustc_version", - "syn", + "syn 2.0.117", ] [[package]] @@ -5704,7 +5715,7 @@ dependencies = [ "bumpalo", "proc-macro2", "quote", - "syn", + "syn 2.0.117", "wasm-bindgen-shared", ] @@ -5857,7 +5868,7 @@ dependencies = [ "darling", "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -5881,7 +5892,7 @@ checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -5892,7 +5903,7 @@ checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -6197,7 +6208,7 @@ dependencies = [ "heck", "indexmap 2.14.0", "prettyplease", - "syn", + "syn 2.0.117", "wasm-metadata", "wit-bindgen-core", "wit-component", @@ -6213,7 +6224,7 @@ dependencies = [ "prettyplease", "proc-macro2", "quote", - "syn", + "syn 2.0.117", "wit-bindgen-core", "wit-bindgen-rust", ] @@ -6307,7 +6318,7 @@ checksum = "de844c262c8848816172cef550288e7dc6c7b7814b4ee56b3e1553f275f1858e" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", "synstructure", ] @@ -6328,7 +6339,7 @@ checksum = "0b631b19d36a892ab55420c92dbc83ccd79274f25be714855d3074aa71cab639" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -6348,7 +6359,7 @@ checksum = "11532158c46691caf0f2593ea8358fed6bbf68a0315e80aae9bd41fbade684a1" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", "synstructure", ] @@ -6388,7 +6399,7 @@ checksum = "625dc425cab0dca6dc3c3319506e6593dcb08a9f387ea3b284dbd52a92c40555" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] diff --git a/MODULE.bazel.lock b/MODULE.bazel.lock index 8fb6a7f22..09812137a 100644 --- a/MODULE.bazel.lock +++ b/MODULE.bazel.lock @@ -1246,6 +1246,7 @@ "strsim_0.11.1": "{\"dependencies\":[],\"features\":{}}", "subtle_2.6.1": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.8\"}],\"features\":{\"const-generics\":[],\"core_hint_black_box\":[],\"default\":[\"std\",\"i128\"],\"i128\":[],\"nightly\":[],\"std\":[]}}", "syn_2.0.117": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"anyhow\",\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"automod\",\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"flate2\",\"req\":\"^1\",\"target\":\"cfg(not(miri))\"},{\"kind\":\"dev\",\"name\":\"insta\",\"req\":\"^1\"},{\"default_features\":false,\"name\":\"proc-macro2\",\"req\":\"^1.0.91\"},{\"default_features\":false,\"name\":\"quote\",\"optional\":true,\"req\":\"^1.0.35\"},{\"kind\":\"dev\",\"name\":\"rayon\",\"req\":\"^1\",\"target\":\"cfg(not(miri))\"},{\"kind\":\"dev\",\"name\":\"ref-cast\",\"req\":\"^1\"},{\"features\":[\"blocking\"],\"kind\":\"dev\",\"name\":\"reqwest\",\"req\":\"^0.13\",\"target\":\"cfg(not(miri))\"},{\"kind\":\"dev\",\"name\":\"rustversion\",\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"syn-test-suite\",\"req\":\"^0\"},{\"kind\":\"dev\",\"name\":\"tar\",\"req\":\"^0.4.16\",\"target\":\"cfg(not(miri))\"},{\"kind\":\"dev\",\"name\":\"termcolor\",\"req\":\"^1\"},{\"name\":\"unicode-ident\",\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"walkdir\",\"req\":\"^2.3.2\",\"target\":\"cfg(not(miri))\"}],\"features\":{\"clone-impls\":[],\"default\":[\"derive\",\"parsing\",\"printing\",\"clone-impls\",\"proc-macro\"],\"derive\":[],\"extra-traits\":[],\"fold\":[],\"full\":[],\"parsing\":[],\"printing\":[\"dep:quote\"],\"proc-macro\":[\"proc-macro2/proc-macro\",\"quote?/proc-macro\"],\"test\":[\"syn-test-suite/all-features\"],\"visit\":[],\"visit-mut\":[]}}", + "syn_3.0.1": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"anyhow\",\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"automod\",\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"flate2\",\"req\":\"^1\",\"target\":\"cfg(not(miri))\"},{\"kind\":\"dev\",\"name\":\"insta\",\"req\":\"^1\"},{\"default_features\":false,\"name\":\"proc-macro2\",\"req\":\"^1.0.91\"},{\"default_features\":false,\"name\":\"quote\",\"optional\":true,\"req\":\"^1.0.35\"},{\"kind\":\"dev\",\"name\":\"rayon\",\"req\":\"^1\",\"target\":\"cfg(not(miri))\"},{\"kind\":\"dev\",\"name\":\"ref-cast\",\"req\":\"^1\"},{\"features\":[\"blocking\"],\"kind\":\"dev\",\"name\":\"reqwest\",\"req\":\"^0.13\",\"target\":\"cfg(not(miri))\"},{\"kind\":\"dev\",\"name\":\"rustversion\",\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"syn-test-suite\",\"req\":\"^0\"},{\"kind\":\"dev\",\"name\":\"tar\",\"req\":\"^0.4.16\",\"target\":\"cfg(not(miri))\"},{\"kind\":\"dev\",\"name\":\"termcolor\",\"req\":\"^1\"},{\"name\":\"unicode-ident\",\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"walkdir\",\"req\":\"^2.3.2\",\"target\":\"cfg(not(miri))\"}],\"features\":{\"clone-impls\":[],\"default\":[\"derive\",\"parsing\",\"printing\",\"clone-impls\",\"proc-macro\"],\"derive\":[],\"extra-traits\":[],\"fold\":[],\"full\":[],\"parsing\":[],\"printing\":[\"dep:quote\"],\"proc-macro\":[\"proc-macro2/proc-macro\",\"quote?/proc-macro\"],\"test\":[\"syn-test-suite/all-features\"],\"visit\":[],\"visit-mut\":[]}}", "sync_wrapper_1.0.2": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"futures\",\"req\":\"^0.3\"},{\"default_features\":false,\"name\":\"futures-core\",\"optional\":true,\"req\":\"^0.3\"},{\"kind\":\"dev\",\"name\":\"pin-project-lite\",\"req\":\"^0.2.7\"}],\"features\":{\"futures\":[\"futures-core\"]}}", "synstructure_0.13.2": "{\"dependencies\":[{\"default_features\":false,\"name\":\"proc-macro2\",\"req\":\"^1.0.60\"},{\"default_features\":false,\"name\":\"quote\",\"req\":\"^1\"},{\"default_features\":false,\"features\":[\"derive\",\"parsing\",\"printing\",\"clone-impls\",\"visit\",\"extra-traits\"],\"name\":\"syn\",\"req\":\"^2\"},{\"kind\":\"dev\",\"name\":\"synstructure_test_traits\",\"req\":\"^0.1\"}],\"features\":{\"default\":[\"proc-macro\"],\"proc-macro\":[\"proc-macro2/proc-macro\",\"syn/proc-macro\",\"quote/proc-macro\"]}}", "system-configuration-sys_0.6.0": "{\"dependencies\":[{\"name\":\"core-foundation-sys\",\"req\":\"^0.8\"},{\"name\":\"libc\",\"req\":\"^0.2.149\"}],\"features\":{}}", diff --git a/nativelink-macro/Cargo.toml b/nativelink-macro/Cargo.toml index 26734588f..c92ca114d 100644 --- a/nativelink-macro/Cargo.toml +++ b/nativelink-macro/Cargo.toml @@ -12,4 +12,9 @@ proc-macro = true [dependencies] proc-macro2 = { version = "1.0.94", default-features = false } quote = { version = "1.0.40", default-features = false } -syn = { version = "2.0.117", default-features = false } +syn = { version = "3.0.0", default-features = false, features = [ + "full", + "parsing", + "printing", + "proc-macro", +] } diff --git a/nativelink-metric/nativelink-metric-macro-derive/Cargo.toml b/nativelink-metric/nativelink-metric-macro-derive/Cargo.toml index 60d9c31c3..490e8e597 100644 --- a/nativelink-metric/nativelink-metric-macro-derive/Cargo.toml +++ b/nativelink-metric/nativelink-metric-macro-derive/Cargo.toml @@ -9,4 +9,9 @@ proc-macro = true [dependencies] proc-macro2 = { version = "1.0.94", default-features = false } quote = { version = "1.0.40", default-features = false } -syn = { version = "2.0.117", default-features = false } +syn = { version = "3.0.0", default-features = false, features = [ + "derive", + "parsing", + "printing", + "proc-macro", +] } diff --git a/nativelink-metric/nativelink-metric-macro-derive/src/lib.rs b/nativelink-metric/nativelink-metric-macro-derive/src/lib.rs index 4488e4a87..b0e1aa969 100644 --- a/nativelink-metric/nativelink-metric-macro-derive/src/lib.rs +++ b/nativelink-metric/nativelink-metric-macro-derive/src/lib.rs @@ -93,7 +93,6 @@ impl ToTokens for MetricKind { } /// Holds general information about a specific field that is to be published. -#[derive(Debug)] struct MetricFieldMetaData<'a> { field_name: &'a Ident, metric_kind: MetricKind, @@ -133,7 +132,6 @@ impl<'a> MetricFieldMetaData<'a> { /// Holds the template information about the struct. This is needed /// to create the `MetricsComponent` impl. -#[derive(Debug)] struct Generics<'a> { implementation: ImplGenerics<'a>, ty: TypeGenerics<'a>, @@ -142,7 +140,6 @@ struct Generics<'a> { /// Holds metadata about the struct that is having `MetricsComponent` /// implemented. -#[derive(Debug)] struct MetricStruct<'a> { name: &'a Ident, metric_fields: Vec>, diff --git a/nativelink-test/fuzz/Cargo.lock b/nativelink-test/fuzz/Cargo.lock index cca966b96..370f76c40 100644 --- a/nativelink-test/fuzz/Cargo.lock +++ b/nativelink-test/fuzz/Cargo.lock @@ -58,7 +58,7 @@ checksum = "9035ad2d096bed7955a320ee7e2230574d28fd3c3a0f186cbea1ff3c7eed5dbb" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.118", ] [[package]] @@ -277,7 +277,7 @@ dependencies = [ "proc-macro2", "quote", "strsim", - "syn", + "syn 2.0.118", ] [[package]] @@ -288,7 +288,7 @@ checksum = "ac3984ec7bd6cfa798e62b4a642426a5be0e68f9401cfc2a01e3fa9ea2fcdb8d" dependencies = [ "darling_core", "quote", - "syn", + "syn 2.0.118", ] [[package]] @@ -305,7 +305,7 @@ checksum = "d65d7ce8132b7c0e54497a4d9a55a1c2a0912a0d786cf894472ba818fba45762" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.118", ] [[package]] @@ -316,7 +316,7 @@ checksum = "d08b3a0bcc0d079199cd476b2cae8435016ec11d1c0986c6901c5ac223041534" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.118", ] [[package]] @@ -338,7 +338,7 @@ dependencies = [ "proc-macro2", "quote", "rustc_version", - "syn", + "syn 2.0.118", "unicode-xid", ] @@ -361,7 +361,7 @@ checksum = "1ac70aa55017e108007fbaf5aa0f54b021c98f92ff8af59d42eda9da96e3dd4f" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.118", ] [[package]] @@ -463,7 +463,7 @@ checksum = "e835b70203e41293343137df5c0664546da5745f82ec9b84d40be8336958447b" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.118", ] [[package]] @@ -897,7 +897,7 @@ dependencies = [ "macro_magic_core", "macro_magic_macros", "quote", - "syn", + "syn 2.0.118", ] [[package]] @@ -911,7 +911,7 @@ dependencies = [ "macro_magic_core_macros", "proc-macro2", "quote", - "syn", + "syn 2.0.118", ] [[package]] @@ -922,7 +922,7 @@ checksum = "b02abfe41815b5bd98dbd4260173db2c116dda171dc0fe7838cb206333b83308" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.118", ] [[package]] @@ -933,7 +933,7 @@ checksum = "73ea28ee64b88876bf45277ed9a5817c1817df061a74f2b988971a12570e5869" dependencies = [ "macro_magic_core", "quote", - "syn", + "syn 2.0.118", ] [[package]] @@ -1033,7 +1033,7 @@ dependencies = [ "macro_magic", "proc-macro2", "quote", - "syn", + "syn 2.0.118", ] [[package]] @@ -1100,7 +1100,7 @@ version = "1.6.2" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 3.0.1", ] [[package]] @@ -1209,7 +1209,7 @@ dependencies = [ "pest_meta", "proc-macro2", "quote", - "syn", + "syn 2.0.118", ] [[package]] @@ -1239,7 +1239,7 @@ checksum = "c96395f0a926bc13b1c17622aaddda1ecb55d49c8f1bf9777e4d877800a43f8b" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.118", ] [[package]] @@ -1301,7 +1301,7 @@ dependencies = [ "itertools", "proc-macro2", "quote", - "syn", + "syn 2.0.118", ] [[package]] @@ -1571,7 +1571,7 @@ checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.118", ] [[package]] @@ -1618,7 +1618,7 @@ dependencies = [ "darling", "proc-macro2", "quote", - "syn", + "syn 2.0.118", ] [[package]] @@ -1727,6 +1727,17 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "syn" +version = "3.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5edbec4ed188954a10c12c038215f8ce7606b2d5c973cd8dc43e8795065c5f2f" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + [[package]] name = "sync_wrapper" version = "1.0.2" @@ -1744,7 +1755,7 @@ checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.118", ] [[package]] @@ -1776,7 +1787,7 @@ checksum = "ebc4ee7f67670e9b64d05fa4253e753e016c6c95ff35b89b7941d6b856dec1d5" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.118", ] [[package]] @@ -1867,7 +1878,7 @@ checksum = "385a6cb71ab9ab790c5fe8d67f1645e6c450a7ce006a33de03daa956cf70a496" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.118", ] [[package]] @@ -2014,7 +2025,7 @@ checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.118", ] [[package]] @@ -2049,7 +2060,7 @@ checksum = "0e48cea23f68d1f78eb7bc092881b6bb88d3d6b5b7e6234f6f9c911da1ffb221" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.118", ] [[package]] @@ -2233,7 +2244,7 @@ dependencies = [ "bumpalo", "proc-macro2", "quote", - "syn", + "syn 2.0.118", "wasm-bindgen-shared", ] @@ -2408,7 +2419,7 @@ checksum = "de844c262c8848816172cef550288e7dc6c7b7814b4ee56b3e1553f275f1858e" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.118", "synstructure", ] @@ -2429,7 +2440,7 @@ checksum = "1ae7f38b72ec2a254e2b87ef277cf2cd4fb97cbebf944faa6f33354da0867930" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.118", ] [[package]] @@ -2449,7 +2460,7 @@ checksum = "11532158c46691caf0f2593ea8358fed6bbf68a0315e80aae9bd41fbade684a1" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.118", "synstructure", ] @@ -2489,7 +2500,7 @@ checksum = "625dc425cab0dca6dc3c3319506e6593dcb08a9f387ea3b284dbd52a92c40555" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.118", ] [[package]] From d1e49ac5f547006c3ecba9e6a94b16ab82897dce Mon Sep 17 00:00:00 2001 From: Tom Parker-Shemilt Date: Mon, 20 Jul 2026 15:33:15 +0100 Subject: [PATCH 23/84] anyhow to 1.0.104 (#2580) --- Cargo.lock | 4 ++-- MODULE.bazel.lock | 2 +- nativelink-test/fuzz/Cargo.lock | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index bcef4a231..a3f60eb46 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -97,9 +97,9 @@ dependencies = [ [[package]] name = "anyhow" -version = "1.0.103" +version = "1.0.104" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2a4385e2e34eb35d6b3efe798b9eb88096925d87726c0798709bf56d9ed84af3" +checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470" [[package]] name = "arc-swap" diff --git a/MODULE.bazel.lock b/MODULE.bazel.lock index 09812137a..cb22fba44 100644 --- a/MODULE.bazel.lock +++ b/MODULE.bazel.lock @@ -786,7 +786,7 @@ "anstyle_1.0.13": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"lexopt\",\"req\":\"^0.3.0\"},{\"kind\":\"dev\",\"name\":\"snapbox\",\"req\":\"^0.6.5\"}],\"features\":{\"default\":[\"std\"],\"std\":[]}}", "anstyle_1.0.14": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"lexopt\",\"req\":\"^0.3.1\"},{\"kind\":\"dev\",\"name\":\"snapbox\",\"req\":\"^0.6.23\"}],\"features\":{\"default\":[\"std\"],\"std\":[]}}", "anyhow_1.0.102": "{\"dependencies\":[{\"default_features\":false,\"kind\":\"dev\",\"name\":\"futures\",\"req\":\"^0.3\"},{\"kind\":\"dev\",\"name\":\"rustversion\",\"req\":\"^1.0.6\"},{\"features\":[\"full\"],\"kind\":\"dev\",\"name\":\"syn\",\"req\":\"^2.0\"},{\"kind\":\"dev\",\"name\":\"thiserror\",\"req\":\"^2\"},{\"features\":[\"diff\"],\"kind\":\"dev\",\"name\":\"trybuild\",\"req\":\"^1.0.108\"}],\"features\":{\"backtrace\":[],\"default\":[\"std\"],\"std\":[]}}", - "anyhow_1.0.103": "{\"dependencies\":[{\"default_features\":false,\"kind\":\"dev\",\"name\":\"futures\",\"req\":\"^0.3\"},{\"kind\":\"dev\",\"name\":\"rustversion\",\"req\":\"^1.0.6\"},{\"features\":[\"full\"],\"kind\":\"dev\",\"name\":\"syn\",\"req\":\"^2.0\"},{\"kind\":\"dev\",\"name\":\"thiserror\",\"req\":\"^2\"},{\"features\":[\"diff\"],\"kind\":\"dev\",\"name\":\"trybuild\",\"req\":\"^1.0.108\"}],\"features\":{\"backtrace\":[],\"default\":[\"std\"],\"std\":[]}}", + "anyhow_1.0.104": "{\"dependencies\":[{\"default_features\":false,\"kind\":\"dev\",\"name\":\"futures\",\"req\":\"^0.3\"},{\"kind\":\"dev\",\"name\":\"rustversion\",\"req\":\"^1.0.6\"},{\"features\":[\"full\"],\"kind\":\"dev\",\"name\":\"syn\",\"req\":\"^3\"},{\"kind\":\"dev\",\"name\":\"thiserror\",\"req\":\"^2\"},{\"features\":[\"diff\"],\"kind\":\"dev\",\"name\":\"trybuild\",\"req\":\"^1.0.108\"}],\"features\":{\"backtrace\":[],\"default\":[\"std\"],\"std\":[]}}", "arc-swap_1.7.1": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"adaptive-barrier\",\"req\":\"~1\"},{\"kind\":\"dev\",\"name\":\"criterion\",\"req\":\"~0.5\"},{\"kind\":\"dev\",\"name\":\"crossbeam-utils\",\"req\":\"~0.8\"},{\"kind\":\"dev\",\"name\":\"itertools\",\"req\":\"^0.12\"},{\"kind\":\"dev\",\"name\":\"num_cpus\",\"req\":\"~1\"},{\"kind\":\"dev\",\"name\":\"once_cell\",\"req\":\"~1\"},{\"kind\":\"dev\",\"name\":\"parking_lot\",\"req\":\"~0.12\"},{\"kind\":\"dev\",\"name\":\"proptest\",\"req\":\"^1\"},{\"features\":[\"rc\"],\"name\":\"serde\",\"optional\":true,\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"serde_derive\",\"req\":\"^1.0.130\"},{\"kind\":\"dev\",\"name\":\"serde_test\",\"req\":\"^1.0.130\"}],\"features\":{\"experimental-strategies\":[],\"experimental-thread-local\":[],\"internal-test-strategies\":[],\"weak\":[]}}", "arcstr_1.2.0": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"loom\",\"req\":\"^0.7.1\",\"target\":\"cfg(loom)\"},{\"default_features\":false,\"name\":\"serde\",\"optional\":true,\"req\":\"^1\"},{\"default_features\":false,\"kind\":\"dev\",\"name\":\"serde_test\",\"req\":\"^1\"}],\"features\":{\"default\":[\"substr\"],\"std\":[],\"substr\":[],\"substr-usize-indices\":[\"substr\"]}}", "arrayref_0.3.9": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"quickcheck\",\"req\":\"^1.0\"}],\"features\":{}}", diff --git a/nativelink-test/fuzz/Cargo.lock b/nativelink-test/fuzz/Cargo.lock index 370f76c40..f2a680193 100644 --- a/nativelink-test/fuzz/Cargo.lock +++ b/nativelink-test/fuzz/Cargo.lock @@ -17,9 +17,9 @@ dependencies = [ [[package]] name = "anyhow" -version = "1.0.102" +version = "1.0.104" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f202df86484c868dbad7eaa557ef785d5c66295e41b460ef922eca0723b842c" +checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470" [[package]] name = "arbitrary" From 45a56520751e3266be7427bd61950a40eeb49ff0 Mon Sep 17 00:00:00 2001 From: Tom Parker-Shemilt Date: Tue, 21 Jul 2026 08:04:33 +0100 Subject: [PATCH 24/84] Redis eviction events support (#2521) * Work towards dealing with redis eviction * Boot redis subscription manager on start * Handle eviction events * Refactor RemoveItemCallback type * Run remove callbacks for Redis * Add redis_store_tester support for Redis eviction events * Fix some build issues * Add redis store eviction tests * Add docs about required new redis config * Better config get parsing * Fix redis key decoding * Test with redis_store plus existence cache --------- Co-authored-by: Marcus Eagan --- integration_tests/generate_redis_keys.py | 6 + nativelink-config/examples/redis.json5 | 4 +- nativelink-config/src/stores.rs | 6 + nativelink-service/tests/bep_server_test.rs | 7 +- nativelink-service/tests/cas_server_test.rs | 7 +- nativelink-store/src/azure_blob_store.rs | 7 +- nativelink-store/src/cache_metrics_store.rs | 7 +- nativelink-store/src/callback_utils.rs | 15 +- .../src/completeness_checking_store.rs | 7 +- nativelink-store/src/compression_store.rs | 7 +- nativelink-store/src/dedup_store.rs | 7 +- nativelink-store/src/existence_cache_store.rs | 7 +- nativelink-store/src/fast_slow_store.rs | 9 +- nativelink-store/src/filesystem_store.rs | 17 +- nativelink-store/src/gcs_store.rs | 7 +- nativelink-store/src/grpc_store.rs | 7 +- nativelink-store/src/memory_store.rs | 13 +- nativelink-store/src/mongo_store.rs | 7 +- nativelink-store/src/noop_store.rs | 7 +- .../src/ontap_s3_existence_cache_store.rs | 7 +- nativelink-store/src/ontap_s3_store.rs | 9 +- nativelink-store/src/redis_store.rs | 197 +++++++--- nativelink-store/src/ref_store.rs | 9 +- nativelink-store/src/s3_store.rs | 9 +- nativelink-store/src/shard_store.rs | 7 +- .../src/size_partitioning_store.rs | 7 +- nativelink-store/src/verify_store.rs | 7 +- .../tests/fast_slow_store_test.rs | 35 +- nativelink-store/tests/redis_store_test.rs | 349 ++++++++++++++++-- nativelink-util/src/store_trait.rs | 12 +- nativelink-util/tests/origin_event_test.rs | 9 +- nativelink-util/tests/store_trait_test.rs | 7 +- src/bin/redis_store_tester.rs | 41 +- .../docs/reference/nativelink-config/main.mdx | 6 + 34 files changed, 614 insertions(+), 251 deletions(-) create mode 100644 integration_tests/generate_redis_keys.py diff --git a/integration_tests/generate_redis_keys.py b/integration_tests/generate_redis_keys.py new file mode 100644 index 000000000..4b954f45c --- /dev/null +++ b/integration_tests/generate_redis_keys.py @@ -0,0 +1,6 @@ +from sys import stderr + + +for i in range(1000000): + print('set name'+str(i),'helloworld') + print(i, file=stderr) diff --git a/nativelink-config/examples/redis.json5 b/nativelink-config/examples/redis.json5 index 082ea6cd6..476ac42a2 100644 --- a/nativelink-config/examples/redis.json5 +++ b/nativelink-config/examples/redis.json5 @@ -6,7 +6,7 @@ addresses: [ "redis://127.0.0.1:6379/", ], - mode: "cluster", + mode: "standard", }, }, { @@ -15,7 +15,7 @@ addresses: [ "redis://127.0.0.1:6379/", ], - mode: "cluster", + mode: "standard", }, }, { diff --git a/nativelink-config/src/stores.rs b/nativelink-config/src/stores.rs index 3d3469861..8aca04747 100644 --- a/nativelink-config/src/stores.rs +++ b/nativelink-config/src/stores.rs @@ -593,6 +593,12 @@ pub enum StoreSpec { /// Ideal for accepting small object sizes as most redis store /// services have a max file upload of between 256Mb-512Mb. /// + /// If you are using Redis together with any stores above it + /// e.g. existence cache, you will need to configure `notify-keyspace-events` + /// to `KA` as per + /// in order for us to get eviction events. Failing to do so will get you + /// log messages complaining about it, as well as errors like + /// /// **Example JSON Config:** /// ```json /// "redis_store": { diff --git a/nativelink-service/tests/bep_server_test.rs b/nativelink-service/tests/bep_server_test.rs index 9d0139408..df8a6a1bb 100644 --- a/nativelink-service/tests/bep_server_test.rs +++ b/nativelink-service/tests/bep_server_test.rs @@ -48,7 +48,7 @@ use nativelink_util::common::encode_stream_proto; use nativelink_util::default_health_status_indicator; use nativelink_util::health_utils::HealthStatusIndicator; use nativelink_util::store_trait::{ - RemoveItemCallback, Store, StoreDriver, StoreKey, StoreLike, UploadSizeInfo, + RemoveCallback, Store, StoreDriver, StoreKey, StoreLike, UploadSizeInfo, }; use pretty_assertions::assert_eq; use prost::Message; @@ -154,10 +154,7 @@ impl StoreDriver for FlakyStore { self } - fn register_remove_callback( - self: Arc, - _callback: Arc, - ) -> Result<(), Error> { + fn register_remove_callback(self: Arc, _callback: RemoveCallback) -> Result<(), Error> { todo!(); } } diff --git a/nativelink-service/tests/cas_server_test.rs b/nativelink-service/tests/cas_server_test.rs index 32b49fd54..c786a7730 100644 --- a/nativelink-service/tests/cas_server_test.rs +++ b/nativelink-service/tests/cas_server_test.rs @@ -44,7 +44,7 @@ use nativelink_util::common::DigestInfo; use nativelink_util::digest_hasher::{DigestHasher, DigestHasherFunc}; use nativelink_util::health_utils::{HealthStatusIndicator, default_health_status_indicator}; use nativelink_util::store_trait::{ - RemoveItemCallback, Store, StoreDriver, StoreKey, StoreLike, UploadSizeInfo, + RemoveCallback, Store, StoreDriver, StoreKey, StoreLike, UploadSizeInfo, }; use pretty_assertions::assert_eq; use prost::Message; @@ -752,10 +752,7 @@ impl StoreDriver for StallStore { self } - fn register_remove_callback( - self: Arc, - _callback: Arc, - ) -> Result<(), Error> { + fn register_remove_callback(self: Arc, _callback: RemoveCallback) -> Result<(), Error> { Ok(()) } } diff --git a/nativelink-store/src/azure_blob_store.rs b/nativelink-store/src/azure_blob_store.rs index 194336c81..0bbdd0d71 100644 --- a/nativelink-store/src/azure_blob_store.rs +++ b/nativelink-store/src/azure_blob_store.rs @@ -42,7 +42,7 @@ use nativelink_util::health_utils::{HealthRegistryBuilder, HealthStatus, HealthS use nativelink_util::instant_wrapper::InstantWrapper; use nativelink_util::retry::{Retrier, RetryResult}; use nativelink_util::store_trait::{ - RemoveItemCallback, StoreDriver, StoreKey, StoreOptimizations, UploadSizeInfo, + RemoveCallback, StoreDriver, StoreKey, StoreOptimizations, UploadSizeInfo, }; use tokio::sync::mpsc; use tokio::time::sleep; @@ -720,10 +720,7 @@ where registry.register_indicator(self); } - fn register_remove_callback( - self: Arc, - _callback: Arc, - ) -> Result<(), Error> { + fn register_remove_callback(self: Arc, _callback: RemoveCallback) -> Result<(), Error> { // Azure Blob Storage manages object lifecycle externally, // so we can safely ignore remove callbacks. Ok(()) diff --git a/nativelink-store/src/cache_metrics_store.rs b/nativelink-store/src/cache_metrics_store.rs index 9c3a7933a..deca501d6 100644 --- a/nativelink-store/src/cache_metrics_store.rs +++ b/nativelink-store/src/cache_metrics_store.rs @@ -31,7 +31,7 @@ use nativelink_util::fs; use nativelink_util::health_utils::{HealthRegistryBuilder, HealthStatusIndicator}; use nativelink_util::metrics::{CACHE_METRICS, CACHE_TYPE, CacheMetricAttrs}; use nativelink_util::store_trait::{ - RemoveItemCallback, Store, StoreDriver, StoreKey, StoreLike, StoreOptimizations, UploadSizeInfo, + RemoveCallback, Store, StoreDriver, StoreKey, StoreLike, StoreOptimizations, UploadSizeInfo, }; use opentelemetry::KeyValue; @@ -275,10 +275,7 @@ impl StoreDriver for CacheMetricsStore { self.backend.clone().register_health(registry); } - fn register_remove_callback( - self: Arc, - callback: Arc, - ) -> Result<(), Error> { + fn register_remove_callback(self: Arc, callback: RemoveCallback) -> Result<(), Error> { self.backend.register_remove_callback(callback) } } diff --git a/nativelink-store/src/callback_utils.rs b/nativelink-store/src/callback_utils.rs index a18f20c52..29e0783c7 100644 --- a/nativelink-store/src/callback_utils.rs +++ b/nativelink-store/src/callback_utils.rs @@ -14,25 +14,24 @@ use core::borrow::Borrow; use core::pin::Pin; -use std::sync::Arc; use nativelink_util::evicting_map; -use nativelink_util::store_trait::{RemoveItemCallback, StoreKey}; +use nativelink_util::store_trait::{RemoveCallback, StoreKey}; -// Generic struct to hold a RemoveItemCallback ref for the purposes +// Generic struct to hold a RemoveCallback ref for the purposes // of a RemoveStateCallback call #[derive(Debug)] -pub struct RemoveItemCallbackHolder { - callback: Arc, +pub struct RemoveCallbackHolder { + callback: RemoveCallback, } -impl RemoveItemCallbackHolder { - pub fn new(callback: Arc) -> Self { +impl RemoveCallbackHolder { + pub fn new(callback: RemoveCallback) -> Self { Self { callback } } } -impl<'a, Q> evicting_map::RemoveItemCallback for RemoveItemCallbackHolder +impl<'a, Q> evicting_map::RemoveItemCallback for RemoveCallbackHolder where Q: Borrow>, { diff --git a/nativelink-store/src/completeness_checking_store.rs b/nativelink-store/src/completeness_checking_store.rs index a7f81f807..38e054cef 100644 --- a/nativelink-store/src/completeness_checking_store.rs +++ b/nativelink-store/src/completeness_checking_store.rs @@ -29,7 +29,7 @@ use nativelink_util::common::DigestInfo; use nativelink_util::health_utils::{HealthStatusIndicator, default_health_status_indicator}; use nativelink_util::metrics_utils::CounterWithTime; use nativelink_util::store_trait::{ - RemoveItemCallback, Store, StoreDriver, StoreKey, StoreLike, UploadSizeInfo, + RemoveCallback, Store, StoreDriver, StoreKey, StoreLike, UploadSizeInfo, }; use parking_lot::Mutex; use tokio::sync::Notify; @@ -398,10 +398,7 @@ impl StoreDriver for CompletenessCheckingStore { self } - fn register_remove_callback( - self: Arc, - callback: Arc, - ) -> Result<(), Error> { + fn register_remove_callback(self: Arc, callback: RemoveCallback) -> Result<(), Error> { self.ac_store.register_remove_callback(callback.clone())?; self.cas_store.register_remove_callback(callback)?; Ok(()) diff --git a/nativelink-store/src/compression_store.rs b/nativelink-store/src/compression_store.rs index 1bf816c7b..9e6e52327 100644 --- a/nativelink-store/src/compression_store.rs +++ b/nativelink-store/src/compression_store.rs @@ -30,7 +30,7 @@ use nativelink_util::buf_channel::{ use nativelink_util::health_utils::{HealthStatusIndicator, default_health_status_indicator}; use nativelink_util::spawn; use nativelink_util::store_trait::{ - RemoveItemCallback, Store, StoreDriver, StoreKey, StoreLike, UploadSizeInfo, + RemoveCallback, Store, StoreDriver, StoreKey, StoreLike, UploadSizeInfo, }; use serde::{Deserialize, Serialize}; use wincode::{SchemaRead, SchemaWrite}; @@ -708,10 +708,7 @@ impl StoreDriver for CompressionStore { self } - fn register_remove_callback( - self: Arc, - callback: Arc, - ) -> Result<(), Error> { + fn register_remove_callback(self: Arc, callback: RemoveCallback) -> Result<(), Error> { self.inner_store.register_remove_callback(callback) } } diff --git a/nativelink-store/src/dedup_store.rs b/nativelink-store/src/dedup_store.rs index bde8ed7c5..d37850f8d 100644 --- a/nativelink-store/src/dedup_store.rs +++ b/nativelink-store/src/dedup_store.rs @@ -27,7 +27,7 @@ use nativelink_util::common::DigestInfo; use nativelink_util::fastcdc::FastCDC; use nativelink_util::health_utils::{HealthStatusIndicator, default_health_status_indicator}; use nativelink_util::store_trait::{ - RemoveItemCallback, Store, StoreDriver, StoreKey, StoreLike, UploadSizeInfo, + RemoveCallback, Store, StoreDriver, StoreKey, StoreLike, UploadSizeInfo, }; use serde::{Deserialize, Serialize}; use tokio_util::codec::FramedRead; @@ -396,10 +396,7 @@ impl StoreDriver for DedupStore { self } - fn register_remove_callback( - self: Arc, - callback: Arc, - ) -> Result<(), Error> { + fn register_remove_callback(self: Arc, callback: RemoveCallback) -> Result<(), Error> { self.index_store .register_remove_callback(callback.clone())?; self.content_store.register_remove_callback(callback)?; diff --git a/nativelink-store/src/existence_cache_store.rs b/nativelink-store/src/existence_cache_store.rs index a26b5796e..ffd2625f1 100644 --- a/nativelink-store/src/existence_cache_store.rs +++ b/nativelink-store/src/existence_cache_store.rs @@ -29,7 +29,7 @@ use nativelink_util::evicting_map::{EvictingMap, LenEntry}; use nativelink_util::health_utils::{HealthStatus, HealthStatusIndicator}; use nativelink_util::instant_wrapper::InstantWrapper; use nativelink_util::store_trait::{ - RemoveItemCallback, Store, StoreDriver, StoreKey, StoreLike, UploadSizeInfo, + RemoveCallback, RemoveItemCallback, Store, StoreDriver, StoreKey, StoreLike, UploadSizeInfo, }; use parking_lot::Mutex; use tracing::{debug, info, trace}; @@ -312,10 +312,7 @@ impl StoreDriver for ExistenceCacheStore { self } - fn register_remove_callback( - self: Arc, - callback: Arc, - ) -> Result<(), Error> { + fn register_remove_callback(self: Arc, callback: RemoveCallback) -> Result<(), Error> { self.inner_store.register_remove_callback(callback) } } diff --git a/nativelink-store/src/fast_slow_store.rs b/nativelink-store/src/fast_slow_store.rs index d7ae49a05..d1f76db7c 100644 --- a/nativelink-store/src/fast_slow_store.rs +++ b/nativelink-store/src/fast_slow_store.rs @@ -34,8 +34,8 @@ use nativelink_util::buf_channel::{ use nativelink_util::fs; use nativelink_util::health_utils::{HealthStatusIndicator, default_health_status_indicator}; use nativelink_util::store_trait::{ - RemoveItemCallback, Store, StoreDriver, StoreKey, StoreLike, StoreOptimizations, - UploadSizeInfo, slow_update_store_with_file, + RemoveCallback, Store, StoreDriver, StoreKey, StoreLike, StoreOptimizations, UploadSizeInfo, + slow_update_store_with_file, }; use parking_lot::Mutex; use tokio::sync::OnceCell; @@ -928,10 +928,7 @@ impl StoreDriver for FastSlowStore { self } - fn register_remove_callback( - self: Arc, - callback: Arc, - ) -> Result<(), Error> { + fn register_remove_callback(self: Arc, callback: RemoveCallback) -> Result<(), Error> { self.fast_store.register_remove_callback(callback.clone())?; self.slow_store.register_remove_callback(callback)?; Ok(()) diff --git a/nativelink-store/src/filesystem_store.rs b/nativelink-store/src/filesystem_store.rs index 03cf08fdc..94220e179 100644 --- a/nativelink-store/src/filesystem_store.rs +++ b/nativelink-store/src/filesystem_store.rs @@ -43,8 +43,10 @@ use nativelink_util::fs::FileSlot; use nativelink_util::health_utils::{HealthRegistryBuilder, HealthStatus, HealthStatusIndicator}; #[cfg(unix)] use nativelink_util::spawn_blocking; +#[cfg(unix)] +use nativelink_util::store_trait::RemoveItemCallback; use nativelink_util::store_trait::{ - RemoveItemCallback, StoreDriver, StoreKey, StoreKeyBorrow, StoreOptimizations, UploadSizeInfo, + RemoveCallback, StoreDriver, StoreKey, StoreKeyBorrow, StoreOptimizations, UploadSizeInfo, }; use tokio::io::{AsyncReadExt, AsyncWriteExt, Take}; use tokio::sync::Semaphore; @@ -52,7 +54,7 @@ use tokio::time::timeout; use tokio_stream::wrappers::ReadDirStream; use tracing::{debug, error, info, trace, warn}; -use crate::callback_utils::RemoveItemCallbackHolder; +use crate::callback_utils::RemoveCallbackHolder; use crate::cas_utils::is_zero_digest; // Default size to allocate memory of the buffer when reading files. @@ -476,7 +478,7 @@ pub fn key_from_file(file_name: &str, file_type: FileType) -> Result = - EvictingMap, Arc, SystemTime, RemoveItemCallbackHolder>; + EvictingMap, Arc, SystemTime, RemoveCallbackHolder>; async fn add_files_to_cache( evicting_map: &FsEvictingMap<'_, Fe>, @@ -894,7 +896,7 @@ impl FilesystemStore { fs::create_dir_all(format!("{executable_dir}/{DIGEST_FOLDER}")) .await .err_tip(|| format!("Failed to create executable dir {executable_dir}"))?; - evicting_map.add_remove_callback(RemoveItemCallbackHolder::new(Arc::new( + evicting_map.add_remove_callback(RemoveCallbackHolder::new(Arc::new( ExecutableVariantRemover { content_path: spec.content_path.clone(), }, @@ -1573,12 +1575,9 @@ impl StoreDriver for FilesystemStore { registry.register_indicator(self); } - fn register_remove_callback( - self: Arc, - callback: Arc, - ) -> Result<(), Error> { + fn register_remove_callback(self: Arc, callback: RemoveCallback) -> Result<(), Error> { self.evicting_map - .add_remove_callback(RemoveItemCallbackHolder::new(callback)); + .add_remove_callback(RemoveCallbackHolder::new(callback)); Ok(()) } } diff --git a/nativelink-store/src/gcs_store.rs b/nativelink-store/src/gcs_store.rs index 247ced282..a33a80560 100644 --- a/nativelink-store/src/gcs_store.rs +++ b/nativelink-store/src/gcs_store.rs @@ -30,7 +30,7 @@ use nativelink_util::health_utils::{HealthRegistryBuilder, HealthStatus, HealthS use nativelink_util::instant_wrapper::InstantWrapper; use nativelink_util::retry::{Retrier, RetryResult}; use nativelink_util::store_trait::{ - RemoveItemCallback, StoreDriver, StoreKey, StoreOptimizations, UploadSizeInfo, + RemoveCallback, StoreDriver, StoreKey, StoreOptimizations, UploadSizeInfo, }; use rand::Rng; use tokio::time::{sleep, timeout}; @@ -473,10 +473,7 @@ where registry.register_indicator(self); } - fn register_remove_callback( - self: Arc, - _callback: Arc, - ) -> Result<(), Error> { + fn register_remove_callback(self: Arc, _callback: RemoveCallback) -> Result<(), Error> { // As we're backed by GCS, this store doesn't actually drop stuff // so we can actually just ignore this Ok(()) diff --git a/nativelink-store/src/grpc_store.rs b/nativelink-store/src/grpc_store.rs index 708850f25..ef6c7fd7e 100644 --- a/nativelink-store/src/grpc_store.rs +++ b/nativelink-store/src/grpc_store.rs @@ -49,7 +49,7 @@ use nativelink_util::proto_stream_utils::{ }; use nativelink_util::resource_info::ResourceInfo; use nativelink_util::retry::{Retrier, RetryResult}; -use nativelink_util::store_trait::{RemoveItemCallback, StoreDriver, StoreKey, UploadSizeInfo}; +use nativelink_util::store_trait::{RemoveCallback, StoreDriver, StoreKey, UploadSizeInfo}; use nativelink_util::telemetry::ClientHeaders; use nativelink_util::{background_spawn, default_health_status_indicator, tls_utils}; use opentelemetry::context::Context; @@ -1423,10 +1423,7 @@ impl StoreDriver for GrpcStore { self } - fn register_remove_callback( - self: Arc, - _callback: Arc, - ) -> Result<(), Error> { + fn register_remove_callback(self: Arc, _callback: RemoveCallback) -> Result<(), Error> { Err(Error::new( Code::Internal, "gRPC stores are incompatible with removal callbacks".to_string(), diff --git a/nativelink-store/src/memory_store.rs b/nativelink-store/src/memory_store.rs index ef7fa6b57..2a71227bc 100644 --- a/nativelink-store/src/memory_store.rs +++ b/nativelink-store/src/memory_store.rs @@ -31,11 +31,11 @@ use nativelink_util::health_utils::{ HealthRegistryBuilder, HealthStatusIndicator, default_health_status_indicator, }; use nativelink_util::store_trait::{ - RemoveItemCallback, StoreDriver, StoreKey, StoreKeyBorrow, StoreOptimizations, UploadSizeInfo, + RemoveCallback, StoreDriver, StoreKey, StoreKeyBorrow, StoreOptimizations, UploadSizeInfo, }; use tracing::warn; -use crate::callback_utils::RemoveItemCallbackHolder; +use crate::callback_utils::RemoveCallbackHolder; use crate::cas_utils::is_zero_digest; #[derive(Clone)] @@ -67,7 +67,7 @@ pub struct MemoryStore { StoreKey<'static>, BytesWrapper, SystemTime, - RemoveItemCallbackHolder, + RemoveCallbackHolder, >, /// The eviction policy's `max_bytes` (0 = unbounded). Cached here so `update` /// can skip writes larger than the entire store budget without buffering @@ -287,12 +287,9 @@ impl StoreDriver for MemoryStore { registry.register_indicator(self); } - fn register_remove_callback( - self: Arc, - callback: Arc, - ) -> Result<(), Error> { + fn register_remove_callback(self: Arc, callback: RemoveCallback) -> Result<(), Error> { self.evicting_map - .add_remove_callback(RemoveItemCallbackHolder::new(callback)); + .add_remove_callback(RemoveCallbackHolder::new(callback)); Ok(()) } } diff --git a/nativelink-store/src/mongo_store.rs b/nativelink-store/src/mongo_store.rs index 3f784c7fd..2d458c7f0 100644 --- a/nativelink-store/src/mongo_store.rs +++ b/nativelink-store/src/mongo_store.rs @@ -33,7 +33,7 @@ use nativelink_util::buf_channel::{DropCloserReadHalf, DropCloserWriteHalf}; use nativelink_util::health_utils::{HealthRegistryBuilder, HealthStatus, HealthStatusIndicator}; use nativelink_util::spawn; use nativelink_util::store_trait::{ - BoolValue, RemoveItemCallback, SchedulerCurrentVersionProvider, SchedulerIndexProvider, + BoolValue, RemoveCallback, SchedulerCurrentVersionProvider, SchedulerIndexProvider, SchedulerStore, SchedulerStoreDataProvider, SchedulerStoreDecodeTo, SchedulerStoreKeyProvider, SchedulerSubscription, SchedulerSubscriptionManager, StoreDriver, StoreKey, UploadSizeInfo, }; @@ -627,10 +627,7 @@ impl StoreDriver for ExperimentalMongoStore { registry.register_indicator(self); } - fn register_remove_callback( - self: Arc, - _callback: Arc, - ) -> Result<(), Error> { + fn register_remove_callback(self: Arc, _callback: RemoveCallback) -> Result<(), Error> { // drop because we don't remove anything from Mongo Ok(()) } diff --git a/nativelink-store/src/noop_store.rs b/nativelink-store/src/noop_store.rs index b7aaa7a8d..98b96a505 100644 --- a/nativelink-store/src/noop_store.rs +++ b/nativelink-store/src/noop_store.rs @@ -23,7 +23,7 @@ use nativelink_metric::{ use nativelink_util::buf_channel::{DropCloserReadHalf, DropCloserWriteHalf}; use nativelink_util::health_utils::{HealthStatusIndicator, default_health_status_indicator}; use nativelink_util::store_trait::{ - RemoveItemCallback, StoreDriver, StoreKey, StoreOptimizations, UploadSizeInfo, + RemoveCallback, StoreDriver, StoreKey, StoreOptimizations, UploadSizeInfo, }; #[derive(Debug, Default, Clone, Copy)] @@ -101,10 +101,7 @@ impl StoreDriver for NoopStore { self } - fn register_remove_callback( - self: Arc, - _callback: Arc, - ) -> Result<(), Error> { + fn register_remove_callback(self: Arc, _callback: RemoveCallback) -> Result<(), Error> { // does nothing, so drop Ok(()) } diff --git a/nativelink-store/src/ontap_s3_existence_cache_store.rs b/nativelink-store/src/ontap_s3_existence_cache_store.rs index 264e18660..e238fdb9b 100644 --- a/nativelink-store/src/ontap_s3_existence_cache_store.rs +++ b/nativelink-store/src/ontap_s3_existence_cache_store.rs @@ -36,7 +36,7 @@ use nativelink_util::instant_wrapper::InstantWrapper; use nativelink_util::metrics_utils::CounterWithTime; use nativelink_util::spawn; use nativelink_util::store_trait::{ - RemoveItemCallback, Store, StoreDriver, StoreKey, StoreLike, UploadSizeInfo, + RemoveCallback, RemoveItemCallback, Store, StoreDriver, StoreKey, StoreLike, UploadSizeInfo, }; use serde::{Deserialize, Serialize}; use tokio::fs; @@ -539,10 +539,7 @@ where self } - fn register_remove_callback( - self: Arc, - callback: Arc, - ) -> Result<(), Error> { + fn register_remove_callback(self: Arc, callback: RemoveCallback) -> Result<(), Error> { self.inner_store.register_remove_callback(callback) } } diff --git a/nativelink-store/src/ontap_s3_store.rs b/nativelink-store/src/ontap_s3_store.rs index 42335d0f3..ad5e03178 100644 --- a/nativelink-store/src/ontap_s3_store.rs +++ b/nativelink-store/src/ontap_s3_store.rs @@ -47,7 +47,7 @@ use nativelink_util::buf_channel::{ use nativelink_util::health_utils::{HealthStatus, HealthStatusIndicator}; use nativelink_util::instant_wrapper::InstantWrapper; use nativelink_util::retry::{Retrier, RetryResult}; -use nativelink_util::store_trait::{RemoveItemCallback, StoreDriver, StoreKey, UploadSizeInfo}; +use nativelink_util::store_trait::{RemoveCallback, StoreDriver, StoreKey, UploadSizeInfo}; use parking_lot::Mutex; use rustls::{ClientConfig, RootCertStore}; use rustls_pki_types::CertificateDer; @@ -74,8 +74,6 @@ const DEFAULT_MAX_RETRY_BUFFER_PER_REQUEST: usize = 20 * 1024 * 1024; // 20MB // Default limit for concurrent part uploads per multipart upload const DEFAULT_MULTIPART_MAX_CONCURRENT_UPLOADS: usize = 10; -type RemoveCallback = Arc; - #[derive(Debug, MetricsComponent)] pub struct OntapS3Store { s3_client: Arc, @@ -767,10 +765,7 @@ where self } - fn register_remove_callback( - self: Arc, - callback: Arc, - ) -> Result<(), Error> { + fn register_remove_callback(self: Arc, callback: RemoveCallback) -> Result<(), Error> { self.remove_callbacks.lock().push(callback); Ok(()) } diff --git a/nativelink-store/src/redis_store.rs b/nativelink-store/src/redis_store.rs index 78e29075c..29686b856 100644 --- a/nativelink-store/src/redis_store.rs +++ b/nativelink-store/src/redis_store.rs @@ -35,14 +35,15 @@ use nativelink_error::{Code, Error, ResultExt, make_err, make_input_err}; use nativelink_metric::MetricsComponent; use nativelink_redis_tester::SubscriptionManagerNotify; use nativelink_util::buf_channel::{DropCloserReadHalf, DropCloserWriteHalf}; +use nativelink_util::common::DigestInfo; use nativelink_util::health_utils::{HealthRegistryBuilder, HealthStatus, HealthStatusIndicator}; -use nativelink_util::spawn; use nativelink_util::store_trait::{ - BoolValue, RemoveItemCallback, SchedulerCurrentVersionProvider, SchedulerIndexProvider, + BoolValue, RemoveCallback, SchedulerCurrentVersionProvider, SchedulerIndexProvider, SchedulerStore, SchedulerStoreDataProvider, SchedulerStoreDecodeTo, SchedulerStoreKeyProvider, SchedulerSubscription, SchedulerSubscriptionManager, StoreDriver, StoreKey, UploadSizeInfo, }; use nativelink_util::task::JoinHandleDropGuard; +use nativelink_util::{background_spawn, spawn}; use parking_lot::{Mutex, RwLock}; use patricia_tree::StringPatriciaMap; use redis::aio::{ConnectionLike, ConnectionManager, ConnectionManagerConfig}; @@ -53,9 +54,11 @@ use redis::{ AsyncCommands, AsyncIter, Client, IntoConnectionInfo, PushInfo, ScanOptions, Script, Value, pipe, }; +use serde::Deserialize; +use serde::de::IntoDeserializer; use tokio::select; use tokio::sync::mpsc::{UnboundedReceiver, UnboundedSender, unbounded_channel}; -use tokio::sync::{OwnedSemaphorePermit, Semaphore}; +use tokio::sync::{OnceCell, OwnedSemaphorePermit, Semaphore}; use tokio::time::{sleep, timeout}; use tokio_stream::wrappers::UnboundedReceiverStream; use tracing::{debug, error, info, trace, warn}; @@ -325,6 +328,7 @@ impl RedisManager for StandardRedisManager } async fn psubscribe(&self, pattern: &str) -> Result<(), Error> { + debug!(pattern, "new psubscribe"); let mut connection = self.get_connection().await?.0; let new_subscription = self.subscriptions.lock().insert(String::from(pattern)); if new_subscription { @@ -334,6 +338,7 @@ impl RedisManager for StandardRedisManager } result?; } + debug!(pattern, new_subscription, "new psubscribe complete"); Ok(()) } } @@ -386,10 +391,7 @@ where max_count_per_cursor: u64, /// A manager for subscriptions to keys in Redis. - subscription_manager: tokio::sync::OnceCell>, - - /// Channel for getting subscription messages - subscriber_channel: Mutex>>, + subscription_manager: Arc, /// Permits to limit inflight Redis requests. Technically only /// limits the calls to `get_client()`, but the requests per client @@ -398,6 +400,11 @@ where /// Per-call ceiling for `check_health` PING. health_check_timeout: Duration, + + /// Have we done a subscribe for messages for `remove_callback` subscribes? + has_remove_callback_subscribe: OnceCell<()>, + + remove_callbacks: Arc>>, } impl Debug for RedisStore @@ -416,7 +423,6 @@ where ) .field("scan_count", &self.scan_count) .field("subscription_manager", &self.subscription_manager) - .field("subscriber_channel", &self.subscriber_channel) .field("client_permits", &self.client_permits) .finish() } @@ -447,6 +453,35 @@ impl Drop for ClientWithPermit { } } +/// Decode a `StoreKey` coming from Redis +pub fn decode_key<'a>( + key_prefix: &String, + encoded_key: Cow<'a, str>, +) -> Result, Error> { + let key_no_prefix = if key_prefix.is_empty() { + encoded_key + } else { + match encoded_key.strip_prefix(key_prefix) { + Some(r) => Cow::from(r.to_string()), + None => { + return Err(make_err!( + Code::InvalidArgument, + "Redis key ({}) is missing prefix ({})", + encoded_key, + key_prefix + )); + } + } + }; + let maybe_digest_info: Result<_, serde::de::value::Error> = + DigestInfo::deserialize(key_no_prefix.clone().into_deserializer()); + if let Ok(digest_info) = maybe_digest_info { + Ok(StoreKey::Digest(digest_info)) + } else { + Ok(StoreKey::Str(key_no_prefix)) + } +} + impl RedisStore where C: ConnectionLike + Clone + Sync, @@ -468,6 +503,15 @@ where connection_manager: M, ) -> Result { info!("Redis index fingerprint: {FINGERPRINT_CREATE_INDEX_HEX}"); + let remove_callbacks = Arc::new(async_lock::Mutex::new(Vec::new())); + let subscription_manager = Arc::new(RedisSubscriptionManager::new( + subscriber_channel, + remove_callbacks.clone(), + key_prefix.clone(), + )); + if let Some(channel) = &pub_sub_channel { + connection_manager.psubscribe(channel).await?; + } Ok(Self { connection_manager, @@ -478,11 +522,12 @@ where read_chunk_size, max_chunk_uploads_per_update, scan_count, - subscription_manager: tokio::sync::OnceCell::new(), - subscriber_channel: Mutex::new(Some(subscriber_channel)), + subscription_manager, client_permits: Arc::new(Semaphore::new(max_client_permits)), max_count_per_cursor, health_check_timeout, + has_remove_callback_subscribe: OnceCell::const_new(), + remove_callbacks, }) } @@ -499,8 +544,8 @@ where }) } - /// Encode a [`StoreKey`] so it can be sent to Redis. - fn encode_key<'a>(&self, key: &'a StoreKey<'a>) -> Cow<'a, str> { + /// Encode a `StoreKey` so it can be sent to Redis. + pub fn encode_key<'a>(&self, key: &'a StoreKey<'a>) -> Cow<'a, str> { let key_body = key.as_str(); if self.key_prefix.is_empty() { key_body @@ -913,19 +958,21 @@ where errors.push(format!("Non-utf8 key {raw_key:?}")); continue; }; - if let Some(key) = str_key.strip_prefix(&self.key_prefix) { - let key = StoreKey::new_str(key); - if range.contains(&key) { - iterations += 1; - if !handler(&key) { - error!("Issue in handler"); - errors.push("Issue in handler".to_string()); + match decode_key(&self.key_prefix, Cow::from(str_key)) { + Ok(key) => { + if range.contains(&key) { + iterations += 1; + if !handler(&key) { + error!("Issue in handler"); + errors.push("Issue in handler".to_string()); + } + } else { + trace!(%key, ?range, "Key not in range"); } - } else { - trace!(%key, ?range, "Key not in range"); } - } else { - errors.push("Key doesn't match prefix".to_string()); + Err(e) => { + errors.push(e.to_string()); + } } } Err(err) @@ -1293,11 +1340,38 @@ where registry.register_indicator(self); } - fn register_remove_callback( - self: Arc, - _callback: Arc, - ) -> Result<(), Error> { - // As redis doesn't drop stuff, we can just ignore this + fn register_remove_callback(self: Arc, callback: RemoveCallback) -> Result<(), Error> { + debug!(?callback, "New callback"); + let local_self = self.clone(); + background_spawn!("remove_callback_subscribe", async move { + self.remove_callbacks.lock().await.push(callback); + if let Err(err) = local_self.clone().has_remove_callback_subscribe + .get_or_try_init(|| async move { + let mut client = local_self.get_client().await?; + let cfg = redis::cmd("CONFIG").arg("GET").arg("notify-keyspace-events").to_owned().query_async::>(&mut client.connection_manager).await.map_err(|e| Error::from(e).append("Parsing notify-keyspace-events"))?; + if cfg.len() != 1 { + warn!(?cfg, "Got multiple items for CONFIG GET, expected one"); + return Err(make_input_err!("Got multiple items for CONFIG GET, expected one")); + } + let events_cfg = &cfg.first().ok_or_else(|| make_err!(Code::InvalidArgument, "Only one item"))?.1; + if events_cfg.is_empty() { + error!("notify-keyspace-events not enabled for Redis, will fail to get remove callbacks"); + } else if !events_cfg.contains('K') { + error!(notify_keyspace_events=events_cfg, "notify-keyspace-events does not contain 'K' so won't get keyspace events we need for eviction events"); + } else if !events_cfg.contains('A') { + error!(notify_keyspace_events=events_cfg, "notify-keyspace-events does not contain 'A' so we won't get eviction events"); + } + // FIXME: Redis events spec appears unreliable, so we subscribe anyways + // It should just need Ke as per https://redis.io/docs/latest/develop/pubsub/keyspace-notifications/ + // but I'm yet to get reliable eviction events out of that + info!(notify_keyspace_events=events_cfg, "Attempting to subscribe to eviction events"); + self.connection_manager.psubscribe("__key*__:*").await?; + Ok::<(), Error>(()) + }) + .await { + error!(?err, "Error while trying to initialise remove_callback_subscribe"); + } + }); Ok(()) } } @@ -1645,7 +1719,11 @@ pub struct RedisSubscriptionManager { } impl RedisSubscriptionManager { - pub fn new(subscriber_channel: UnboundedReceiver) -> Self { + pub fn new( + subscriber_channel: UnboundedReceiver, + remove_callbacks: Arc>>, + key_prefix: String, + ) -> Self { let subscribed_keys = Arc::new(RwLock::new(StringPatriciaMap::new())); let subscribed_keys_weak = Arc::downgrade(&subscribed_keys); let (tx_for_test, mut rx_for_test) = unbounded_channel(); @@ -1656,6 +1734,7 @@ impl RedisSubscriptionManager { _subscription_spawn: Arc::new(Mutex::new(spawn!( "redis_subscribe_spawn", async move { + debug!("running subscribe loop"); loop { loop { let key = select! { @@ -1682,7 +1761,7 @@ impl RedisSubscriptionManager { error!(?push_info, "Expected exactly 3 values on subscriber channel (pattern, channel, value)"); continue; } - match push_info.data.last().unwrap() { + let value = match push_info.data.last().unwrap() { Value::SimpleString(s) => { s.clone() } @@ -1693,7 +1772,42 @@ impl RedisSubscriptionManager { error!(?other, "Received non-string message in RedisSubscriptionManager"); continue; } + }; + if value == "evicted" { + trace!(?push_info, "Eviction event"); + let eviction_key = if let Some(key) = push_info.data.get(1) { + if let Value::BulkString(s) = key { + String::from_utf8(s.clone()).expect("String message") + } else { + error!(?push_info, "Eviction key wasn't bulk-string"); + continue; + } + } else { + error!(?push_info, "No key in eviction event"); + continue; + }; + trace!(?eviction_key, "Eviction key"); + let Some((_prefix, internal_key)) = eviction_key.split_once(':') else { + error!(?eviction_key, "Eviction key doesn't contain a colon"); + continue; + }; + + let store_key = match decode_key(&key_prefix, Cow::from(internal_key)) { + Ok(k) => k.into_owned(), + Err(err) => { + error!(%err, internal_key, "Bad redis key"); + continue; + } + }; + let locked_remove_callbacks = remove_callbacks.lock().await; + let mut callbacks: FuturesUnordered<_> = + locked_remove_callbacks.iter() + .map(|callback| callback.callback(store_key.borrow())) + .collect(); + while callbacks.next().await.is_some() {} + continue } + value } else { error!("Error receiving message in RedisSubscriptionManager from subscriber_channel"); break; @@ -1786,23 +1900,12 @@ where type SubscriptionManager = RedisSubscriptionManager; async fn subscription_manager(&self) -> Result, Error> { - self.subscription_manager - .get_or_try_init(|| async move { - let Some(subscriber_channel) = self.subscriber_channel.lock().take() else { - return Err(make_input_err!( - "Multiple attempts to obtain the subscription manager in RedisStore" - )); - }; - let Some(pub_sub_channel) = &self.pub_sub_channel else { - return Err(make_input_err!( - "RedisStore must have a pubsub for Redis Scheduler if using subscriptions" - )); - }; - self.connection_manager.psubscribe(pub_sub_channel).await?; - Ok(Arc::new(RedisSubscriptionManager::new(subscriber_channel))) - }) - .await - .map(Clone::clone) + if self.pub_sub_channel.is_none() { + return Err(make_input_err!( + "RedisStore must have a pubsub for Redis Scheduler if using subscriptions" + )); + } + Ok(self.subscription_manager.clone()) } async fn update_data(&self, data: T, expiry: Option) -> Result, Error> diff --git a/nativelink-store/src/ref_store.rs b/nativelink-store/src/ref_store.rs index c6100db8f..26c15678b 100644 --- a/nativelink-store/src/ref_store.rs +++ b/nativelink-store/src/ref_store.rs @@ -23,7 +23,7 @@ use nativelink_metric::MetricsComponent; use nativelink_util::buf_channel::{DropCloserReadHalf, DropCloserWriteHalf}; use nativelink_util::health_utils::{HealthStatusIndicator, default_health_status_indicator}; use nativelink_util::store_trait::{ - RemoveItemCallback, Store, StoreDriver, StoreKey, StoreLike, UploadSizeInfo, + RemoveCallback, Store, StoreDriver, StoreKey, StoreLike, UploadSizeInfo, }; use parking_lot::Mutex; use tracing::{debug, error}; @@ -48,7 +48,7 @@ pub struct RefStore { name: String, store_manager: Weak, inner: StoreReference, - remove_callbacks: Mutex>>, + remove_callbacks: Mutex>, } impl RefStore { @@ -159,10 +159,7 @@ impl StoreDriver for RefStore { self } - fn register_remove_callback( - self: Arc, - callback: Arc, - ) -> Result<(), Error> { + fn register_remove_callback(self: Arc, callback: RemoveCallback) -> Result<(), Error> { self.remove_callbacks.lock().push(callback.clone()); let ref_store = self.inner.cell.0.get(); unsafe { diff --git a/nativelink-store/src/s3_store.rs b/nativelink-store/src/s3_store.rs index 5c2d3837e..f201109ec 100644 --- a/nativelink-store/src/s3_store.rs +++ b/nativelink-store/src/s3_store.rs @@ -47,7 +47,7 @@ use nativelink_util::health_utils::{HealthRegistryBuilder, HealthStatus, HealthS use nativelink_util::instant_wrapper::InstantWrapper; use nativelink_util::retry::{Retrier, RetryResult}; use nativelink_util::store_trait::{ - RemoveItemCallback, StoreDriver, StoreKey, StoreOptimizations, UploadSizeInfo, + RemoveCallback, StoreDriver, StoreKey, StoreOptimizations, UploadSizeInfo, }; use parking_lot::Mutex; use tokio::sync::mpsc; @@ -98,7 +98,7 @@ pub struct S3Store { #[metric(help = "The number of concurrent uploads allowed for multipart uploads")] multipart_max_concurrent_uploads: usize, - remove_callbacks: Mutex>>, + remove_callbacks: Mutex>, } impl S3Store @@ -684,10 +684,7 @@ where registry.register_indicator(self); } - fn register_remove_callback( - self: Arc, - callback: Arc, - ) -> Result<(), Error> { + fn register_remove_callback(self: Arc, callback: RemoveCallback) -> Result<(), Error> { self.remove_callbacks.lock().push(callback); Ok(()) } diff --git a/nativelink-store/src/shard_store.rs b/nativelink-store/src/shard_store.rs index 4c21cdca6..2f2494481 100644 --- a/nativelink-store/src/shard_store.rs +++ b/nativelink-store/src/shard_store.rs @@ -27,7 +27,7 @@ use nativelink_metric::MetricsComponent; use nativelink_util::buf_channel::{DropCloserReadHalf, DropCloserWriteHalf}; use nativelink_util::health_utils::{HealthStatusIndicator, default_health_status_indicator}; use nativelink_util::store_trait::{ - RemoveItemCallback, Store, StoreDriver, StoreKey, StoreLike, UploadSizeInfo, + RemoveCallback, Store, StoreDriver, StoreKey, StoreLike, UploadSizeInfo, }; #[derive(Debug, MetricsComponent)] @@ -254,10 +254,7 @@ impl StoreDriver for ShardStore { self } - fn register_remove_callback( - self: Arc, - callback: Arc, - ) -> Result<(), Error> { + fn register_remove_callback(self: Arc, callback: RemoveCallback) -> Result<(), Error> { for store in &self.weights_and_stores { store.store.register_remove_callback(callback.clone())?; } diff --git a/nativelink-store/src/size_partitioning_store.rs b/nativelink-store/src/size_partitioning_store.rs index 883530078..b9883e423 100644 --- a/nativelink-store/src/size_partitioning_store.rs +++ b/nativelink-store/src/size_partitioning_store.rs @@ -23,7 +23,7 @@ use nativelink_metric::MetricsComponent; use nativelink_util::buf_channel::{DropCloserReadHalf, DropCloserWriteHalf}; use nativelink_util::health_utils::{HealthStatusIndicator, default_health_status_indicator}; use nativelink_util::store_trait::{ - RemoveItemCallback, Store, StoreDriver, StoreKey, StoreLike, UploadSizeInfo, + RemoveCallback, Store, StoreDriver, StoreKey, StoreLike, UploadSizeInfo, }; use tokio::join; @@ -171,10 +171,7 @@ impl StoreDriver for SizePartitioningStore { self } - fn register_remove_callback( - self: Arc, - callback: Arc, - ) -> Result<(), Error> { + fn register_remove_callback(self: Arc, callback: RemoveCallback) -> Result<(), Error> { self.lower_store .register_remove_callback(callback.clone())?; self.upper_store.register_remove_callback(callback)?; diff --git a/nativelink-store/src/verify_store.rs b/nativelink-store/src/verify_store.rs index 1b313e08e..117ef5277 100644 --- a/nativelink-store/src/verify_store.rs +++ b/nativelink-store/src/verify_store.rs @@ -27,7 +27,7 @@ use nativelink_util::digest_hasher::{DigestHasher, digest_hasher_func_from_conte use nativelink_util::health_utils::{HealthStatusIndicator, default_health_status_indicator}; use nativelink_util::metrics_utils::CounterWithTime; use nativelink_util::store_trait::{ - RemoveItemCallback, Store, StoreDriver, StoreKey, StoreLike, UploadSizeInfo, + RemoveCallback, Store, StoreDriver, StoreKey, StoreLike, UploadSizeInfo, }; #[derive(Debug, MetricsComponent)] @@ -235,10 +235,7 @@ impl StoreDriver for VerifyStore { self } - fn register_remove_callback( - self: Arc, - callback: Arc, - ) -> Result<(), Error> { + fn register_remove_callback(self: Arc, callback: RemoveCallback) -> Result<(), Error> { self.inner_store.register_remove_callback(callback) } } diff --git a/nativelink-store/tests/fast_slow_store_test.rs b/nativelink-store/tests/fast_slow_store_test.rs index 769c489e6..b6ee7ee5a 100644 --- a/nativelink-store/tests/fast_slow_store_test.rs +++ b/nativelink-store/tests/fast_slow_store_test.rs @@ -33,7 +33,7 @@ use nativelink_util::buf_channel::{ use nativelink_util::common::DigestInfo; use nativelink_util::health_utils::{HealthStatusIndicator, default_health_status_indicator}; use nativelink_util::store_trait::{ - RemoveItemCallback, Store, StoreDriver, StoreKey, StoreLike, UploadSizeInfo, + RemoveCallback, Store, StoreDriver, StoreKey, StoreLike, UploadSizeInfo, }; use pretty_assertions::assert_eq; use rand::rngs::SmallRng; @@ -323,7 +323,7 @@ async fn drop_on_eof_completes_store_futures() -> Result<(), Error> { fn register_remove_callback( self: Arc, - _callback: Arc, + _callback: RemoveCallback, ) -> Result<(), Error> { Ok(()) } @@ -654,7 +654,7 @@ fn make_stores_with_lazy_slow() -> (Store, Store, Store) { fn register_remove_callback( self: Arc, - _callback: Arc, + _callback: RemoveCallback, ) -> Result<(), Error> { Ok(()) } @@ -794,10 +794,7 @@ impl StoreDriver for InstrumentedSlowStore { self } - fn register_remove_callback( - self: Arc, - _callback: Arc, - ) -> Result<(), Error> { + fn register_remove_callback(self: Arc, _callback: RemoveCallback) -> Result<(), Error> { Ok(()) } } @@ -1050,7 +1047,7 @@ async fn has_sees_in_flight_slow_writes() -> Result<(), Error> { fn register_remove_callback( self: Arc, - _callback: Arc, + _callback: RemoveCallback, ) -> Result<(), Error> { Ok(()) } @@ -1210,7 +1207,7 @@ async fn has_does_not_consult_fast_store_when_slow_store_hits() -> Result<(), Er fn register_remove_callback( self: Arc, - _callback: Arc, + _callback: RemoveCallback, ) -> Result<(), Error> { Ok(()) } @@ -1316,10 +1313,7 @@ impl StoreDriver for GatedSlowStore2 { fn as_any_arc(self: Arc) -> Arc { self } - fn register_remove_callback( - self: Arc, - _callback: Arc, - ) -> Result<(), Error> { + fn register_remove_callback(self: Arc, _callback: RemoveCallback) -> Result<(), Error> { Ok(()) } } @@ -1476,10 +1470,7 @@ impl StoreDriver for MapBackedSlow { fn as_any_arc(self: Arc) -> Arc { self } - fn register_remove_callback( - self: Arc, - _cb: Arc, - ) -> Result<(), Error> { + fn register_remove_callback(self: Arc, _cb: RemoveCallback) -> Result<(), Error> { Ok(()) } } @@ -1658,10 +1649,7 @@ impl StoreDriver for CountingSlowStore { fn as_any_arc(self: Arc) -> Arc { self } - fn register_remove_callback( - self: Arc, - _callback: Arc, - ) -> Result<(), Error> { + fn register_remove_callback(self: Arc, _callback: RemoveCallback) -> Result<(), Error> { Ok(()) } } @@ -1922,10 +1910,7 @@ impl StoreDriver for StaleFastStore { self } - fn register_remove_callback( - self: Arc, - _callback: Arc, - ) -> Result<(), Error> { + fn register_remove_callback(self: Arc, _callback: RemoveCallback) -> Result<(), Error> { Ok(()) } } diff --git a/nativelink-store/tests/redis_store_test.rs b/nativelink-store/tests/redis_store_test.rs index 2bdac314c..2f463b30a 100644 --- a/nativelink-store/tests/redis_store_test.rs +++ b/nativelink-store/tests/redis_store_test.rs @@ -13,13 +13,14 @@ // limitations under the License. use core::ops::RangeBounds; +use core::pin::Pin; use core::time::Duration; use std::collections::HashMap; use std::sync::Arc; use bytes::{Bytes, BytesMut}; use futures::TryStreamExt; -use nativelink_config::stores::{RedisMode, RedisSpec}; +use nativelink_config::stores::{ExistenceCacheSpec, RedisMode, RedisSpec, StoreSpec}; use nativelink_error::{Code, Error, ErrorContext, ResultExt, make_err}; use nativelink_macro::nativelink_test; use nativelink_redis_tester::{ @@ -28,22 +29,24 @@ use nativelink_redis_tester::{ make_fake_redis_with_responses, }; use nativelink_store::cas_utils::ZERO_BYTE_DIGESTS; +use nativelink_store::existence_cache_store::ExistenceCacheStore; use nativelink_store::redis_store::{ ClusterRedisManager, DEFAULT_MAX_CHUNK_UPLOADS_PER_UPDATE, DEFAULT_MAX_COUNT_PER_CURSOR, - LUA_VERSION_SET_SCRIPT, RedisStore, RedisSubscriptionManager, + LUA_VERSION_SET_SCRIPT, RedisStore, RedisSubscriptionManager, decode_key, }; use nativelink_util::buf_channel::make_buf_channel_pair; use nativelink_util::common::DigestInfo; use nativelink_util::health_utils::HealthStatus; use nativelink_util::store_trait::{ - FalseValue, SchedulerCurrentVersionProvider, SchedulerIndexProvider, SchedulerStore, - SchedulerStoreDataProvider, SchedulerStoreDecodeTo, SchedulerStoreKeyProvider, - SchedulerSubscription, SchedulerSubscriptionManager, StoreKey, StoreLike, TrueValue, - UploadSizeInfo, + FalseValue, RemoveItemCallback, SchedulerCurrentVersionProvider, SchedulerIndexProvider, + SchedulerStore, SchedulerStoreDataProvider, SchedulerStoreDecodeTo, SchedulerStoreKeyProvider, + SchedulerSubscription, SchedulerSubscriptionManager, Store, StoreDriver, StoreKey, StoreLike, + TrueValue, UploadSizeInfo, }; use pretty_assertions::assert_eq; use redis::{PushInfo, RedisError, Value, make_extension_error}; use redis_test::{MockCmd, MockRedisConnection}; +use tokio::sync::mpsc::UnboundedReceiver; use tokio::time::{sleep, timeout}; use tracing::{Instrument, info, info_span}; @@ -84,9 +87,10 @@ async fn fake_redis_sentinel_master_stream_with_script() -> u16 { .await } -async fn make_mock_store_with_prefix( +async fn make_mock_store_with_prefix_and_subscriber_channel( mut commands: Vec, key_prefix: String, + subscriber_channel: UnboundedReceiver, ) -> RedisStore> { commands.insert( 0, @@ -97,7 +101,6 @@ async fn make_mock_store_with_prefix( ); let mock_connection = MockRedisConnection::new(commands); let manager = ClusterRedisManager::new(mock_connection).await.unwrap(); - let (_tx, rx) = tokio::sync::mpsc::unbounded_channel(); RedisStore::new_from_builder_and_parts( None, mock_uuid_generator, @@ -108,13 +111,21 @@ async fn make_mock_store_with_prefix( DEFAULT_MAX_PERMITS, DEFAULT_MAX_COUNT_PER_CURSOR, Duration::from_secs(4), - rx, + subscriber_channel, manager, ) .await .unwrap() } +async fn make_mock_store_with_prefix( + commands: Vec, + key_prefix: String, +) -> RedisStore> { + let (_tx, rx) = tokio::sync::mpsc::unbounded_channel(); + make_mock_store_with_prefix_and_subscriber_channel(commands, key_prefix, rx).await +} + #[nativelink_test] async fn upload_and_get_data() -> Result<(), Error> { // Construct the data we want to send. Since it's small, we expect it to be sent in a single chunk. @@ -133,20 +144,18 @@ async fn upload_and_get_data() -> Result<(), Error> { // Append the real value to the temp key. MockCmd::new( redis::cmd("SETRANGE") - .arg(temp_key.clone()) + .arg(&temp_key) .arg(0) .arg(data.to_vec()), Ok(Value::Int(0)), ), MockCmd::new( - redis::cmd("STRLEN").arg(temp_key.clone()), + redis::cmd("STRLEN").arg(&temp_key), Ok(Value::Int(data.len() as i64)), ), // Move the data from the fake key to the real key. MockCmd::new( - redis::cmd("RENAME") - .arg(temp_key.clone()) - .arg(real_key.clone()), + redis::cmd("RENAME").arg(&temp_key).arg(&real_key), Ok(Value::Nil), ), // The second set of commands are for retrieving the data from the key. @@ -154,9 +163,9 @@ async fn upload_and_get_data() -> Result<(), Error> { MockCmd::with_values( redis::pipe() .cmd("STRLEN") - .arg(real_key.clone()) + .arg(&real_key) .cmd("EXISTS") - .arg(real_key.clone()), + .arg(&real_key), Ok(vec![Value::Int(2), Value::Boolean(true)]), ), // Retrieve the data from the real key. @@ -1863,7 +1872,8 @@ fn test_search_by_index_skips_int_from_cursor_read() -> Result<(), Error> { #[nativelink_test] async fn no_items_from_none_subscription_channel() -> Result<(), Error> { let (_tx, rx) = tokio::sync::mpsc::unbounded_channel(); - let subscription_manager = RedisSubscriptionManager::new(rx); + let subscription_manager = + RedisSubscriptionManager::new(rx, Arc::new(async_lock::Mutex::new(vec![])), String::new()); // To give the stream enough time to get polled sleep(Duration::from_secs(1)).await; @@ -1882,7 +1892,8 @@ async fn no_items_from_none_subscription_channel() -> Result<(), Error> { #[nativelink_test] async fn send_messages_to_subscription_channel() -> Result<(), Error> { let (tx, rx) = tokio::sync::mpsc::unbounded_channel(); - let subscription_manager = RedisSubscriptionManager::new(rx); + let subscription_manager = + RedisSubscriptionManager::new(rx, Arc::new(async_lock::Mutex::new(vec![])), String::new()); tx.send(PushInfo { kind: redis::PushKind::PSubscribe, @@ -1922,6 +1933,10 @@ async fn send_messages_to_subscription_channel() -> Result<(), Error> { // Because otherwise it gets dropped immediately, and we need it to live to do things drop(subscription_manager); + assert!(logs_contain( + "PSubscribe, ignore push_info=PushInfo { kind: PSubscribe, data: [bulk-string('\"scheduler_key_change\"'), int(1)] }" + )); + Ok(()) } @@ -2041,7 +2056,8 @@ impl SchedulerStoreKeyProvider for TestSubKey { #[nativelink_test] async fn redis_subscription_single_drop_is_silent() -> Result<(), Error> { let (_tx, rx) = tokio::sync::mpsc::unbounded_channel(); - let manager = RedisSubscriptionManager::new(rx); + let manager = + RedisSubscriptionManager::new(rx, Arc::new(async_lock::Mutex::new(vec![])), String::new()); let sub = manager.subscribe(TestSubKey("solo-key".to_string()))?; drop(sub); @@ -2060,7 +2076,8 @@ async fn redis_subscription_single_drop_is_silent() -> Result<(), Error> { #[nativelink_test] async fn redis_subscription_drop_one_of_two_keeps_publisher() -> Result<(), Error> { let (_tx, rx) = tokio::sync::mpsc::unbounded_channel(); - let manager = RedisSubscriptionManager::new(rx); + let manager = + RedisSubscriptionManager::new(rx, Arc::new(async_lock::Mutex::new(vec![])), String::new()); let key = "shared-key"; let sub_a = manager.subscribe(TestSubKey(key.to_string()))?; @@ -2088,7 +2105,8 @@ async fn redis_subscription_drop_one_of_two_keeps_publisher() -> Result<(), Erro async fn redis_subscription_concurrent_drops_no_absence_warn() -> Result<(), Error> { const ITERATIONS: usize = 200; let (_tx, rx) = tokio::sync::mpsc::unbounded_channel(); - let manager = RedisSubscriptionManager::new(rx); + let manager = + RedisSubscriptionManager::new(rx, Arc::new(async_lock::Mutex::new(vec![])), String::new()); for i in 0..ITERATIONS { let key = format!("race-key-{i}"); @@ -2122,7 +2140,8 @@ async fn redis_subscription_concurrent_drops_no_absence_warn() -> Result<(), Err #[nativelink_test] async fn redis_subscription_resubscribe_after_drop_creates_fresh_publisher() -> Result<(), Error> { let (_tx, rx) = tokio::sync::mpsc::unbounded_channel(); - let manager = RedisSubscriptionManager::new(rx); + let manager = + RedisSubscriptionManager::new(rx, Arc::new(async_lock::Mutex::new(vec![])), String::new()); let key = "cycle-key"; let sub_a = manager.subscribe(TestSubKey(key.to_string()))?; @@ -2148,3 +2167,289 @@ async fn redis_subscription_resubscribe_after_drop_creates_fresh_publisher() -> drop(manager); Ok(()) } + +#[derive(Debug)] +struct LoggingRemoveCallback {} + +impl RemoveItemCallback for LoggingRemoveCallback { + fn callback<'a>( + &'a self, + store_key: StoreKey<'a>, + ) -> Pin + Send + 'a>> { + info!(?store_key, "Callback for removed item"); + Box::pin(async {}) + } +} + +async fn callback_for_eviction_core( + logs_contain: F, + notify_keyspace_events: &[u8], +) -> Result<(), Error> +where + F: Fn(&str) -> bool, +{ + let redis_span = info_span!("redis"); + + let mut responses = add_lua_version_script(fake_redis_stream()); + add_to_response( + &mut responses, + redis::cmd("CONFIG") + .arg("GET") + .arg("notify-keyspace-events"), + vec![Value::Map(vec![( + Value::BulkString(b"notify-keyspace-events".into()), + Value::BulkString(notify_keyspace_events.into()), + )])], + ); + add_to_response( + &mut responses, + redis::cmd("PSUBSCRIBE").arg("__key*__:*"), + vec![Value::Nil], + ); + let redis_port = make_fake_redis_with_responses(responses) + .instrument(redis_span) + .await; + let spec = RedisSpec { + addresses: vec![format!("redis://127.0.0.1:{redis_port}/")], + mode: RedisMode::Standard, + ..Default::default() + }; + let mut raw_store = + Arc::into_inner(RedisStore::new_standard(spec).await.expect("Working spec")).unwrap(); + raw_store.replace_temp_name_generator(mock_uuid_generator); + let store = Arc::new(raw_store); + store.register_remove_callback(Arc::new(LoggingRemoveCallback {}))?; + + timeout(Duration::from_secs(3), async move { + loop { + if logs_contain("new psubscribe complete pattern=\"__key*__:*\" new_subscription=true") + { + break; + } + sleep(Duration::from_millis(100)).await; + } + }) + .await?; + Ok(()) +} + +#[nativelink_test] +async fn callback_for_eviction_not_enabled() -> Result<(), Error> { + callback_for_eviction_core(logs_contain, b"").await?; + assert!(logs_contain( + "notify-keyspace-events not enabled for Redis, will fail to get remove callbacks" + )); + Ok(()) +} + +#[nativelink_test] +async fn callback_for_eviction_no_keyspace() -> Result<(), Error> { + callback_for_eviction_core(logs_contain, b"E").await?; + assert!(logs_contain( + "notify-keyspace-events does not contain 'K' so won't get keyspace events we need for eviction events notify_keyspace_events=\"E\"" + )); + Ok(()) +} + +#[nativelink_test] +async fn callback_for_eviction_no_all() -> Result<(), Error> { + callback_for_eviction_core(logs_contain, b"K").await?; + assert!(logs_contain( + "notify-keyspace-events does not contain 'A' so we won't get eviction events notify_keyspace_events=\"K\"" + )); + Ok(()) +} + +#[nativelink_test] +async fn callback_for_eviction_good() -> Result<(), Error> { + callback_for_eviction_core(logs_contain, b"KA").await?; + assert!(!logs_contain("ERROR")); + Ok(()) +} + +#[nativelink_test] +async fn send_eviction_to_subscription_channel() -> Result<(), Error> { + let (tx, rx) = tokio::sync::mpsc::unbounded_channel(); + let subscription_manager = RedisSubscriptionManager::new( + rx, + Arc::new(async_lock::Mutex::new(vec![Arc::new( + LoggingRemoveCallback {}, + )])), + String::new(), + ); + + tx.send(PushInfo { + kind: redis::PushKind::PMessage, + data: vec![ + Value::BulkString("demo_pattern".into()), + Value::BulkString("keyprefix:test-eviction".into()), + Value::BulkString("evicted".into()), + ], + }) + .unwrap(); + + timeout(Duration::from_secs(5), async { + loop { + assert!(!logs_contain("ERROR")); + if logs_contain("Callback for removed item store_key=Str(\"test-eviction\")") { + break; + } + sleep(Duration::from_millis(100)).await; + } + }) + .await + .unwrap(); + + // Because otherwise it gets dropped immediately, and we need it to live to do things + drop(subscription_manager); + + assert!(logs_contain( + "Eviction key eviction_key=\"keyprefix:test-eviction\"" + )); + + Ok(()) +} + +async fn store_key_coding_round_trip_core(key_prefix: String) -> Result<(), Error> { + let store = make_mock_store_with_prefix(vec![], key_prefix.clone()).await; + + for key in [ + StoreKey::new_str("foo"), + StoreKey::Digest(DigestInfo::zero_digest()), + StoreKey::Digest(DigestInfo::new([99u8; 32], 512)), + ] { + assert_eq!(key, decode_key(&key_prefix, store.encode_key(&key))?); + } + Ok(()) +} + +#[nativelink_test] +async fn store_key_coding_round_trip() -> Result<(), Error> { + store_key_coding_round_trip_core(String::new()).await +} + +#[nativelink_test] +async fn store_key_coding_round_trip_with_prefix() -> Result<(), Error> { + store_key_coding_round_trip_core(String::from("demo")).await +} + +async fn evict_keys_for_existence_cache_core( + prefix: String, + logs_contain: F, +) -> Result<(), Error> +where + F: Fn(&str) -> bool + Send + Sync, +{ + let spec = ExistenceCacheSpec { + backend: StoreSpec::RedisStore(RedisSpec::default()), // Note: Not used. + eviction_policy: Option::default(), // Not evicting here, we'll evict from Redis + }; + let (tx, rx) = tokio::sync::mpsc::unbounded_channel(); + + let data = Bytes::from_static(b"14"); + let digest = DigestInfo::try_new(VALID_HASH1, 2)?; + let real_key = format!("{prefix}{digest}"); + let temp_key = make_temp_key(&real_key); + + let commands = vec![ + MockCmd::new( + redis::cmd("CONFIG") + .arg("GET") + .arg("notify-keyspace-events"), + Ok(Value::Map(vec![( + Value::BulkString(b"notify-keyspace-events".into()), + Value::BulkString(b"KA".into()), + )])), + ), + MockCmd::with_values( + redis::pipe() + .cmd("STRLEN") + .arg(&real_key) + .cmd("EXISTS") + .arg(&real_key), + Ok(vec![Value::Int(2), Value::Boolean(true)]), + ), + MockCmd::new( + redis::cmd("SETRANGE") + .arg(&temp_key) + .arg(0) + .arg(data.to_vec()), + Ok(Value::Int(0)), + ), + MockCmd::new( + redis::cmd("STRLEN").arg(&temp_key), + Ok(Value::Int(data.len() as i64)), + ), + // Move the data from the fake key to the real key. + MockCmd::new( + redis::cmd("RENAME").arg(&temp_key).arg(&real_key), + Ok(Value::Nil), + ), + // Retrieve the data from the real key. + MockCmd::new( + redis::cmd("GETRANGE").arg(&real_key).arg(0).arg(1), + Ok(Value::BulkString(b"14".to_vec())), + ), + ]; + + let redis_store = Arc::new( + make_mock_store_with_prefix_and_subscriber_channel(commands, prefix.clone(), rx).await, + ); + let existence_store = ExistenceCacheStore::new(&spec, Store::new(redis_store.clone())); + + // Wait for subscription first so the mock redis commands are in the right order + timeout(Duration::from_secs(5), async { + loop { + assert!(!logs_contain("ERROR")); + if logs_contain("Attempting to subscribe to eviction events") { + break; + } + sleep(Duration::from_millis(100)).await; + } + }) + .await + .unwrap(); + + existence_store + .update_oneshot(digest, data.clone()) + .await + .unwrap(); + + tx.send(PushInfo { + kind: redis::PushKind::PMessage, + data: vec![ + Value::BulkString("demo_pattern".into()), + Value::BulkString(format!("keyprefix:{real_key}").into()), + Value::BulkString("evicted".into()), + ], + }) + .unwrap(); + + timeout(Duration::from_secs(5), async { + loop { + assert!(!logs_contain("ERROR")); + if logs_contain("Evicting (direct remove) key=DigestInfo(\"3031323334353637383961626364656630303030303030303030303030303030-2\")") { + break; + } + sleep(Duration::from_millis(100)).await; + } + }) + .await + .unwrap(); + + assert!(logs_contain(&format!( + "Eviction key eviction_key=\"keyprefix:{prefix}3031323334353637383961626364656630303030303030303030303030303030-2\"" + ))); + + Ok(()) +} + +#[nativelink_test] +async fn evict_keys_for_existence_cache_no_prefix() -> Result<(), Error> { + evict_keys_for_existence_cache_core(String::new(), logs_contain).await +} + +#[nativelink_test] +async fn evict_keys_for_existence_cache_prefix() -> Result<(), Error> { + evict_keys_for_existence_cache_core(String::from("demo_prefix:"), logs_contain).await +} diff --git a/nativelink-util/src/store_trait.rs b/nativelink-util/src/store_trait.rs index 942d3a15c..5fb650e2b 100644 --- a/nativelink-util/src/store_trait.rs +++ b/nativelink-util/src/store_trait.rs @@ -401,10 +401,7 @@ impl Store { } #[inline] - pub fn register_remove_callback( - &self, - callback: Arc, - ) -> Result<(), Error> { + pub fn register_remove_callback(&self, callback: RemoveCallback) -> Result<(), Error> { self.inner.clone().register_remove_callback(callback) } } @@ -617,6 +614,8 @@ pub trait StoreLike: Send + Sync + Sized + Unpin + 'static { } } +pub type RemoveCallback = Arc; + #[async_trait] pub trait StoreDriver: Sync + Send + Unpin + MetricsComponent + HealthStatusIndicator + 'static @@ -865,10 +864,7 @@ pub trait StoreDriver: // Register health checks used to monitor the store. fn register_health(self: Arc, _registry: &mut HealthRegistryBuilder) {} - fn register_remove_callback( - self: Arc, - callback: Arc, - ) -> Result<(), Error>; + fn register_remove_callback(self: Arc, callback: RemoveCallback) -> Result<(), Error>; } // Callback to be called when a store deletes an item. This is used so diff --git a/nativelink-util/tests/origin_event_test.rs b/nativelink-util/tests/origin_event_test.rs index 5404ad278..9f72e7d05 100644 --- a/nativelink-util/tests/origin_event_test.rs +++ b/nativelink-util/tests/origin_event_test.rs @@ -29,9 +29,7 @@ use nativelink_util::default_health_status_indicator; use nativelink_util::health_utils::HealthStatusIndicator; use nativelink_util::origin_event::get_id_for_event; use nativelink_util::origin_event_publisher::OriginEventPublisher; -use nativelink_util::store_trait::{ - RemoveItemCallback, Store, StoreDriver, StoreKey, UploadSizeInfo, -}; +use nativelink_util::store_trait::{RemoveCallback, Store, StoreDriver, StoreKey, UploadSizeInfo}; use tokio::sync::{broadcast, mpsc}; use tokio::time::sleep; use tonic::async_trait; @@ -100,10 +98,7 @@ impl StoreDriver for FlakyStore { self } - fn register_remove_callback( - self: Arc, - _callback: Arc, - ) -> Result<(), Error> { + fn register_remove_callback(self: Arc, _callback: RemoveCallback) -> Result<(), Error> { todo!(); } } diff --git a/nativelink-util/tests/store_trait_test.rs b/nativelink-util/tests/store_trait_test.rs index c09b03838..0cecc7ee3 100644 --- a/nativelink-util/tests/store_trait_test.rs +++ b/nativelink-util/tests/store_trait_test.rs @@ -8,7 +8,7 @@ use nativelink_util::buf_channel::{DropCloserReadHalf, DropCloserWriteHalf}; use nativelink_util::default_health_status_indicator; use nativelink_util::health_utils::HealthStatusIndicator; use nativelink_util::store_trait::{ - RemoveItemCallback, Store, StoreDriver, StoreKey, StoreLike, UploadSizeInfo, + RemoveCallback, Store, StoreDriver, StoreKey, StoreLike, UploadSizeInfo, }; use tonic::async_trait; @@ -61,10 +61,7 @@ impl StoreDriver for FakeStore { self } - fn register_remove_callback( - self: Arc, - _callback: Arc, - ) -> Result<(), Error> { + fn register_remove_callback(self: Arc, _callback: RemoveCallback) -> Result<(), Error> { todo!(); } } diff --git a/src/bin/redis_store_tester.rs b/src/bin/redis_store_tester.rs index b5048161f..4d63757d2 100644 --- a/src/bin/redis_store_tester.rs +++ b/src/bin/redis_store_tester.rs @@ -1,3 +1,4 @@ +use core::pin::Pin; use core::sync::atomic::{AtomicUsize, Ordering}; use core::time::Duration; use std::borrow::Cow; @@ -12,7 +13,7 @@ use nativelink_error::{Code, Error, ResultExt}; use nativelink_store::redis_store::RedisStore; use nativelink_util::buf_channel::make_buf_channel_pair; use nativelink_util::store_trait::{ - SchedulerCurrentVersionProvider, SchedulerIndexProvider, SchedulerStore, + RemoveItemCallback, SchedulerCurrentVersionProvider, SchedulerIndexProvider, SchedulerStore, SchedulerStoreDataProvider, SchedulerStoreDecodeTo, SchedulerStoreKeyProvider, StoreDriver, StoreKey, StoreLike, TrueValue, UploadSizeInfo, }; @@ -155,6 +156,11 @@ async fn run( ) -> Result<(), Error> { let mut count = 0; let in_flight = Arc::new(AtomicUsize::new(0)); + let fixed_action_value = if let Ok(str_action_value) = env::var("ACTION_VALUE") { + Some(str::parse::(&str_action_value).unwrap()) + } else { + None + }; loop { if count % 1000 == 0 { @@ -187,9 +193,13 @@ async fn run( let local_in_flight = in_flight.clone(); let max_action_value = 7; - let action_value = match mode { - TestMode::Random => rand::rng().random_range(0..max_action_value), - TestMode::Sequential => count % max_action_value, + let action_value = if let Some(av) = fixed_action_value { + av + } else { + match mode { + TestMode::Random => rand::rng().random_range(0..max_action_value), + TestMode::Sequential => count % max_action_value, + } }; background_spawn!("action", async move { @@ -222,9 +232,11 @@ async fn run( .await?; } 3 => { + let key = random_key(); store_clone - .update_oneshot(random_key(), Bytes::from_static(b"1234")) + .update_oneshot(key.borrow(), Bytes::from_static(b"1234")) .await?; + info!(?key, "Updated"); } 4 => { let res = store_clone @@ -284,6 +296,19 @@ async fn run( } } +#[derive(Debug)] +struct LoggingRemoveCallback {} + +impl RemoveItemCallback for LoggingRemoveCallback { + fn callback<'a>( + &'a self, + store_key: StoreKey<'a>, + ) -> Pin + Send + 'a>> { + info!(?store_key, "Callback for removed item"); + Box::pin(async {}) + } +} + fn main() -> Result<(), Box> { let args = Args::parse(); let redis_mode: RedisMode = args.redis_mode.into(); @@ -330,10 +355,16 @@ fn main() -> Result<(), Box> { match spec.mode { RedisMode::Standard | RedisMode::Sentinel => { let store = RedisStore::new_standard(spec).await?; + store + .clone() + .register_remove_callback(Arc::new(LoggingRemoveCallback {}))?; run(store, max_loops, failed.clone(), args.mode).await } RedisMode::Cluster => { let store = RedisStore::new_cluster(spec).await?; + store + .clone() + .register_remove_callback(Arc::new(LoggingRemoveCallback {}))?; run(store, max_loops, failed.clone(), args.mode).await } } diff --git a/web/apps/docs/content/docs/reference/nativelink-config/main.mdx b/web/apps/docs/content/docs/reference/nativelink-config/main.mdx index 330ea9482..8e094bcfe 100644 --- a/web/apps/docs/content/docs/reference/nativelink-config/main.mdx +++ b/web/apps/docs/content/docs/reference/nativelink-config/main.mdx @@ -618,6 +618,12 @@ Pairs well with `SizePartitioning` and/or `FastSlow` stores. Ideal for accepting small object sizes as most Redis store services have a max file upload of between 256Mb-512Mb. +If you are using Redis together with any stores above it +e.g. existence cache, you will need to configure `notify-keyspace-events` +to `KA` as per [https://redis.io/docs/latest/develop/pubsub/keyspace-notifications/#configuration](https://redis.io/docs/latest/develop/pubsub/keyspace-notifications/#configuration) +in order for us to get eviction events. Failing to do so will get you +log messages complaining about it, as well as errors like [https://github.com/TraceMachina/nativelink/issues/2436](https://github.com/TraceMachina/nativelink/issues/2436) + **Example JSON5 config:** ```json5 "redis_store": { From d59e50f29cff85de3b6194e0bce2a65911830ca4 Mon Sep 17 00:00:00 2001 From: Tom Parker-Shemilt Date: Tue, 21 Jul 2026 12:31:08 +0100 Subject: [PATCH 25/84] cargo-llvm-cov gets flagged, but doesn't have actual vulnerabilities (#2585) --- tools/cargo-llvm-cov/osv-scanner.toml | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 tools/cargo-llvm-cov/osv-scanner.toml diff --git a/tools/cargo-llvm-cov/osv-scanner.toml b/tools/cargo-llvm-cov/osv-scanner.toml new file mode 100644 index 000000000..143a6e642 --- /dev/null +++ b/tools/cargo-llvm-cov/osv-scanner.toml @@ -0,0 +1,9 @@ +# Ignore false positives as per https://github.com/taiki-e/cargo-llvm-cov/blob/main/.deny.toml + +[[IgnoredVulns]] +id = "RUSTSEC-2026-0194" +reason = "quick-xml (via lcov2cobertura). Not affected since lcov2cobertura emits xml, but doesn't parse." + +[[IgnoredVulns]] +id = "RUSTSEC-2026-0195" +reason = "quick-xml (via lcov2cobertura). Not affected since lcov2cobertura doesn't use affected APIs." From 3947c71603b5d3acb39d2f0caf2a22db6b009089 Mon Sep 17 00:00:00 2001 From: Jacob Martin Date: Tue, 21 Jul 2026 11:15:54 -0400 Subject: [PATCH 26/84] Let GCS get_part defer NotFound to the retrier (#2584) A read that failed with NotFound returned RetryResult::Err, which bypassed the retrier entirely. This made a read-404 terminal even though the store's retry config can classify NotFound as retryable via retry_on_errors, so read-after-write races (an object still finalizing or being repopulated after eviction) could never be retried at the store level and had to be absorbed by scheduler re-dispatch, exhausting max_job_retries under concurrent load. Emit RetryResult::Retry instead and let the retrier classify the error. Default behavior is unchanged: the retrier does not retry NotFound unless retry_on_errors opts in, and has() still maps NotFound to Ok(None) for existence probes. Fixes #2582 --- nativelink-store/src/gcs_store.rs | 8 +- nativelink-store/tests/gcs_store_test.rs | 106 ++++++++++++++++++++++- 2 files changed, 110 insertions(+), 4 deletions(-) diff --git a/nativelink-store/src/gcs_store.rs b/nativelink-store/src/gcs_store.rs index a33a80560..ca47ba029 100644 --- a/nativelink-store/src/gcs_store.rs +++ b/nativelink-store/src/gcs_store.rs @@ -429,9 +429,11 @@ where .await { Ok(stream) => stream, - Err(e) if e.code == Code::NotFound => { - return Some((RetryResult::Err(e), (offset, writer))); - } + // NotFound is intentionally not special-cased here: + // the retrier doesn't retry NotFound by default, but + // emitting `Retry` lets `retry.retry_on_errors` opt + // reads into retrying read-after-write races where + // an object is still finalizing or being repopulated. Err(e) => return Some((RetryResult::Retry(e), (offset, writer))), }; diff --git a/nativelink-store/tests/gcs_store_test.rs b/nativelink-store/tests/gcs_store_test.rs index 34f9cba4d..536d40ab2 100644 --- a/nativelink-store/tests/gcs_store_test.rs +++ b/nativelink-store/tests/gcs_store_test.rs @@ -18,7 +18,7 @@ use std::sync::Arc; use bytes::{BufMut, Bytes, BytesMut}; use mock_instant::thread_local::MockClock; -use nativelink_config::stores::{CommonObjectSpec, ExperimentalGcsSpec}; +use nativelink_config::stores::{CommonObjectSpec, ErrorCode, ExperimentalGcsSpec, Retry}; use nativelink_error::{Code, Error, make_err}; use nativelink_macro::nativelink_test; use nativelink_store::cas_utils::ZERO_BYTE_DIGESTS; @@ -181,6 +181,90 @@ async fn get_part_handles_not_found_error() -> Result<(), Error> { Ok(()) } +#[nativelink_test] +async fn get_part_not_found_not_retried_by_default() -> Result<(), Error> { + // Create mock GCS operations without adding any object, so reads + // yield NotFound. + let mock_ops = Arc::new(MockGcsOperations::new()); + let store = create_test_store_with_retry( + mock_ops.clone(), + Retry { + max_retries: 2, + delay: 0.001, + jitter: 0.0, + ..Default::default() + }, + ) + .await?; + + let digest = DigestInfo::try_new(VALID_HASH1, 100)?; + let store_key: StoreKey = to_store_key(digest); + let (mut tx, _rx) = make_buf_channel_pair(); + + let store_clone = store.clone(); + let get_part_fut = nativelink_util::spawn!("get_part_task", async move { + store_clone.get_part(store_key, &mut tx, 0, None).await + }); + + let err = get_part_fut.await?.unwrap_err(); + assert_eq!(err.code, Code::NotFound, "Expected NotFound error"); + + // Even with a retry budget configured, NotFound must not consume it + // unless explicitly opted in via `retry_on_errors`. + let call_counts = mock_ops.get_call_counts(); + assert_eq!( + call_counts.read_calls.load(Ordering::Relaxed), + 1, + "read_object_content should not be retried on NotFound by default" + ); + + Ok(()) +} + +#[nativelink_test] +async fn get_part_retries_not_found_when_opted_in() -> Result<(), Error> { + // Create mock GCS operations without adding any object, so reads + // yield NotFound. + let mock_ops = Arc::new(MockGcsOperations::new()); + let store = create_test_store_with_retry( + mock_ops.clone(), + Retry { + max_retries: 2, + delay: 0.001, + jitter: 0.0, + retry_on_errors: Some(vec![ErrorCode::NotFound]), + }, + ) + .await?; + + let digest = DigestInfo::try_new(VALID_HASH1, 100)?; + let store_key: StoreKey = to_store_key(digest); + let (mut tx, _rx) = make_buf_channel_pair(); + + let store_clone = store.clone(); + let get_part_fut = nativelink_util::spawn!("get_part_task", async move { + store_clone.get_part(store_key, &mut tx, 0, None).await + }); + + let err = get_part_fut.await?.unwrap_err(); + assert_eq!( + err.code, + Code::NotFound, + "Expected NotFound error after retries are exhausted" + ); + + // With NotFound in `retry_on_errors`, the read should be attempted + // once plus `max_retries` more times before giving up. + let call_counts = mock_ops.get_call_counts(); + assert_eq!( + call_counts.read_calls.load(Ordering::Relaxed), + 3, + "read_object_content should be retried on NotFound when opted in" + ); + + Ok(()) +} + #[nativelink_test] async fn has_with_results_test() -> Result<(), Error> { // Create mock GCS operations @@ -705,6 +789,26 @@ async fn create_test_store( ) } +// Helper function to create a test GCS store with a custom retry config +async fn create_test_store_with_retry( + ops: Arc, + retry: Retry, +) -> Result MockInstantWrapped>>, Error> { + GcsStore::new_with_ops( + &ExperimentalGcsSpec { + bucket: BUCKET_NAME.to_string(), + common: CommonObjectSpec { + key_prefix: Some(KEY_PREFIX.to_string()), + retry, + ..Default::default() + }, + ..Default::default() + }, + ops, + MockInstantWrapped::default, + ) +} + // Helper function to create a test GCS store with expiration async fn create_test_store_with_expiration( ops: Arc, From 43c0658a65e75c073dddf575c3ef6cb8e9afcfe9 Mon Sep 17 00:00:00 2001 From: Marcus Eagan Date: Tue, 21 Jul 2026 17:11:57 +0100 Subject: [PATCH 27/84] Give fork PRs read-only remote cache via a public key variable (#2587) --- .../setup-nativelink-cloud/action.yaml | 50 +++++++++++++++++-- .github/workflows/lre.yaml | 2 + .github/workflows/native-bazel.yaml | 4 ++ .github/workflows/nix.yaml | 2 + .github/workflows/sanitizers.yaml | 2 + 5 files changed, 57 insertions(+), 3 deletions(-) diff --git a/.github/actions/setup-nativelink-cloud/action.yaml b/.github/actions/setup-nativelink-cloud/action.yaml index fc34e6b90..6b5b1664f 100644 --- a/.github/actions/setup-nativelink-cloud/action.yaml +++ b/.github/actions/setup-nativelink-cloud/action.yaml @@ -30,6 +30,22 @@ inputs: empty. required: false default: '' + public-read-key: + description: >- + World-readable cache_read-only API key, normally passed from + vars.NATIVELINK_PUBLIC_CACHE_KEY. Used only when api-key/claim-base are + unavailable (fork PRs have no secrets), giving fork contributors + read-only cache hits. Deliberately a variable rather than a secret: + fork code runs with the value in hand either way, so it is public by + definition and must never carry more than cache_read. + required: false + default: '' + public-claim-base: + description: >- + Claim endpoint base for the public read-only path, normally passed from + vars.NATIVELINK_PUBLIC_CLAIM_BASE. + required: false + default: '' ci-mode: description: >- Kill switch, normally passed as vars.NATIVELINK_CI_MODE. Empty or 'off' @@ -73,6 +89,8 @@ runs: NL_API_KEY: ${{ inputs.api-key }} NL_CLAIM_BASE: ${{ inputs.claim-base }} NL_BES_RESULTS_URL: ${{ inputs.bes-results-url }} + NL_PUBLIC_KEY: ${{ inputs.public-read-key }} + NL_PUBLIC_CLAIM_BASE: ${{ inputs.public-claim-base }} NL_CI_MODE: ${{ inputs.ci-mode }} NL_MODE: ${{ inputs.mode }} NL_EXEC: ${{ inputs.exec }} @@ -80,9 +98,18 @@ runs: run: | set -euo pipefail - if [[ -z "$NL_API_KEY" || -z "$NL_CLAIM_BASE" || -z "$NL_CI_MODE" || "$NL_CI_MODE" == "off" ]]; then + cred_kind="" + if [[ -n "$NL_CI_MODE" && "$NL_CI_MODE" != "off" ]]; then + if [[ -n "$NL_API_KEY" && -n "$NL_CLAIM_BASE" ]]; then + cred_kind="private" + elif [[ -n "$NL_PUBLIC_KEY" && -n "$NL_PUBLIC_CLAIM_BASE" ]]; then + cred_kind="public" + fi + fi + + if [[ -z "$cred_kind" ]]; then echo "enabled=false" >> "$GITHUB_OUTPUT" - echo "NativeLink Cloud disabled (no API key/claim or ci-mode off); building locally." + echo "NativeLink Cloud disabled (no credentials or ci-mode off); building locally." exit 0 fi @@ -94,11 +121,28 @@ runs: echo "exec: on is only supported on Linux runners (got $RUNNER_OS)" >&2 exit 1 fi - if [[ "$NL_CI_MODE" == "rbe" ]]; then + # Remote execution needs a write-capable identity; the public + # read-only path never executes remotely. + if [[ "$NL_CI_MODE" == "rbe" && "$cred_kind" == "private" ]]; then exec_enabled=true fi fi + if [[ "$cred_kind" == "public" ]]; then + { + echo "common --config=nl-cache" + echo "common --remote_cache=grpcs://cas-${NL_PUBLIC_CLAIM_BASE}" + # The public key carries cache_read only: no uploads, no BES. + echo "common --remote_upload_local_results=false" + echo "common --remote_header=x-nativelink-api-key=${NL_PUBLIC_KEY}" + echo "common --remote_header=x-nativelink-project=nativelink-github-ci" + echo "common --build_metadata=ROLE=CI" + } >> user.bazelrc + echo "enabled=true" >> "$GITHUB_OUTPUT" + echo "NativeLink Cloud enabled: public read-only cache (no uploads, no BES)" + exit 0 + fi + { echo "common --config=nl-cache" echo "common --remote_cache=grpcs://cas-${NL_CLAIM_BASE}" diff --git a/.github/workflows/lre.yaml b/.github/workflows/lre.yaml index a9229cc12..b83963a3c 100644 --- a/.github/workflows/lre.yaml +++ b/.github/workflows/lre.yaml @@ -48,6 +48,8 @@ jobs: ci-mode: ${{ vars.NATIVELINK_CI_MODE }} claim-base: ${{ secrets.NATIVELINK_STAGING_CLAIM_BASE }} bes-results-url: ${{ secrets.NATIVELINK_STAGING_BES_RESULTS_URL }} + public-read-key: ${{ vars.NATIVELINK_PUBLIC_CACHE_KEY }} + public-claim-base: ${{ vars.NATIVELINK_PUBLIC_CLAIM_BASE }} api-key: >- ${{ github.event_name == 'push' && secrets.NATIVELINK_STAGING_API_KEY_RW diff --git a/.github/workflows/native-bazel.yaml b/.github/workflows/native-bazel.yaml index 7161dab46..e4e02db87 100644 --- a/.github/workflows/native-bazel.yaml +++ b/.github/workflows/native-bazel.yaml @@ -66,6 +66,8 @@ jobs: ci-mode: ${{ vars.NATIVELINK_CI_MODE }} claim-base: ${{ secrets.NATIVELINK_STAGING_CLAIM_BASE }} bes-results-url: ${{ secrets.NATIVELINK_STAGING_BES_RESULTS_URL }} + public-read-key: ${{ vars.NATIVELINK_PUBLIC_CACHE_KEY }} + public-claim-base: ${{ vars.NATIVELINK_PUBLIC_CLAIM_BASE }} api-key: >- ${{ github.event_name == 'push' && secrets.NATIVELINK_STAGING_API_KEY_RW @@ -133,6 +135,8 @@ jobs: ci-mode: ${{ vars.NATIVELINK_CI_MODE }} claim-base: ${{ secrets.NATIVELINK_STAGING_CLAIM_BASE }} bes-results-url: ${{ secrets.NATIVELINK_STAGING_BES_RESULTS_URL }} + public-read-key: ${{ vars.NATIVELINK_PUBLIC_CACHE_KEY }} + public-claim-base: ${{ vars.NATIVELINK_PUBLIC_CLAIM_BASE }} api-key: >- ${{ github.event_name == 'push' && secrets.NATIVELINK_STAGING_API_KEY_RW diff --git a/.github/workflows/nix.yaml b/.github/workflows/nix.yaml index 31a7fae90..a9ad30ef5 100644 --- a/.github/workflows/nix.yaml +++ b/.github/workflows/nix.yaml @@ -44,6 +44,8 @@ jobs: ci-mode: ${{ vars.NATIVELINK_CI_MODE }} claim-base: ${{ secrets.NATIVELINK_STAGING_CLAIM_BASE }} bes-results-url: ${{ secrets.NATIVELINK_STAGING_BES_RESULTS_URL }} + public-read-key: ${{ vars.NATIVELINK_PUBLIC_CACHE_KEY }} + public-claim-base: ${{ vars.NATIVELINK_PUBLIC_CLAIM_BASE }} api-key: >- ${{ github.event_name == 'push' && secrets.NATIVELINK_STAGING_API_KEY_RW diff --git a/.github/workflows/sanitizers.yaml b/.github/workflows/sanitizers.yaml index 2eb7aa65c..8a1bb59bf 100644 --- a/.github/workflows/sanitizers.yaml +++ b/.github/workflows/sanitizers.yaml @@ -50,6 +50,8 @@ jobs: ci-mode: ${{ vars.NATIVELINK_CI_MODE }} claim-base: ${{ secrets.NATIVELINK_STAGING_CLAIM_BASE }} bes-results-url: ${{ secrets.NATIVELINK_STAGING_BES_RESULTS_URL }} + public-read-key: ${{ vars.NATIVELINK_PUBLIC_CACHE_KEY }} + public-claim-base: ${{ vars.NATIVELINK_PUBLIC_CLAIM_BASE }} api-key: >- ${{ github.event_name == 'push' && secrets.NATIVELINK_STAGING_API_KEY_RW From 3b1e179b835b03633ee2b42d4ef4fd6627bb57dd Mon Sep 17 00:00:00 2001 From: Marcus Eagan Date: Tue, 21 Jul 2026 19:26:00 +0100 Subject: [PATCH 28/84] Exclude the aggregate total runner entry from the canary hit-rate math (#2589) --- .github/workflows/nativelink-cloud-canary.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/nativelink-cloud-canary.yaml b/.github/workflows/nativelink-cloud-canary.yaml index 60763c4ef..2539ac7de 100644 --- a/.github/workflows/nativelink-cloud-canary.yaml +++ b/.github/workflows/nativelink-cloud-canary.yaml @@ -81,8 +81,10 @@ jobs: hits=$(jq -s '[.[] | .buildMetrics.actionSummary.runnerCount[]? | select(.name == "remote cache hit") | .count] | add // 0' \ "$RUNNER_TEMP/bep.json") + # runnerCount includes an aggregate "total" entry alongside the + # per-runner entries; count only real runners or the rate is halved. total=$(jq -s '[.[] | .buildMetrics.actionSummary.runnerCount[]? - | select(.name != "internal") | .count] | add // 0' \ + | select(.name != "internal" and .name != "total") | .count] | add // 0' \ "$RUNNER_TEMP/bep.json") invocation=$(jq -rs '[.[] | select(.id.started != null) | .started.uuid][0] // empty' "$RUNNER_TEMP/bep.json") From ddcb47e5e2c0baaa9537fe5c4be87499f44cde24 Mon Sep 17 00:00:00 2001 From: Tom Parker-Shemilt Date: Tue, 21 Jul 2026 19:44:17 +0100 Subject: [PATCH 29/84] cmake links; rust headers; sandboxing (#2586) Co-authored-by: Marcus Eagan --- README.md | 2 +- templates/README.md | 2 +- templates/cmake/README.md | 2 +- .../content/docs/configuration/production.mdx | 20 +++++++++++++++++++ web/apps/docs/content/docs/faq/rust.mdx | 8 ++++---- 5 files changed, 27 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 0469a288e..3c6c96a73 100644 --- a/README.md +++ b/README.md @@ -53,7 +53,7 @@ NativeLink is trusted in production environments to reduce costs and developer i - Utilizes remote resources to offload computational burden from local machines - Ensures consistency with a uniform, controlled build environment -NativeLink seamlessly integrates with build tools that use the Remote Execution protocol, such as [Bazel](https://bazel.build), [Buck2](https://buck2.build), [Goma](https://chromium.googlesource.com/infra/goma/client/), and [Siso](https://chromium.googlesource.com/build/+/refs/heads/main/siso/README.md). CMake projects work too via [`recc`](https://buildgrid.gitlab.io/recc). See [Build CMake projects with NativeLink](https://nativelink.com/docs/rbe/cmake-recc). It supports Unix-based operating systems and Windows, ensuring broad compatibility across different development environments. +NativeLink seamlessly integrates with build tools that use the Remote Execution protocol, such as [Bazel](https://bazel.build), [Buck2](https://buck2.build), [Goma](https://chromium.googlesource.com/infra/goma/client/), and [Siso](https://chromium.googlesource.com/build/+/refs/heads/main/siso/README.md). CMake projects work too via [`recc`](https://buildgrid.gitlab.io/recc). See [Build CMake projects with NativeLink](https://docs.nativelink.com/getting-started/other-build-systems/cmake-recc). It supports Unix-based operating systems and Windows, ensuring broad compatibility across different development environments. ## 🚀 Quickstart diff --git a/templates/README.md b/templates/README.md index 2ec132444..56a0ec268 100644 --- a/templates/README.md +++ b/templates/README.md @@ -10,7 +10,7 @@ NativeLink provides the following templates to use caching and remote execution: C/C++ with CMake using [`recc`](https://buildgrid.gitlab.io/recc) as the bridge to NativeLink. Cache-only by default. Compiles run locally and only their outputs travel through the cache. Works on Linux and macOS. - Tutorial: [Build CMake projects with NativeLink](https://nativelink.com/docs/rbe/cmake-recc). + Tutorial: [Build CMake projects with NativeLink](https://docs.nativelink.com/getting-started/other-build-systems/cmake-recc). # Getting started diff --git a/templates/cmake/README.md b/templates/cmake/README.md index 13d18b1b2..272399627 100644 --- a/templates/cmake/README.md +++ b/templates/cmake/README.md @@ -1,7 +1,7 @@ # CMake template The full tutorial lives at -[Build CMake projects with NativeLink](https://nativelink.com/docs/rbe/cmake-recc). +[Build CMake projects with NativeLink](https://docs.nativelink.com/getting-started/other-build-systems/cmake-recc). This directory holds the example sources (`CMakeLists.txt`, `main.cpp`) that the tutorial references. Copy them into a fresh project directory diff --git a/web/apps/docs/content/docs/configuration/production.mdx b/web/apps/docs/content/docs/configuration/production.mdx index f5f7e2f0b..baae5e22d 100644 --- a/web/apps/docs/content/docs/configuration/production.mdx +++ b/web/apps/docs/content/docs/configuration/production.mdx @@ -210,6 +210,26 @@ What's worth alerting on: The [Metrics deployment guide](/deployment/metrics) has a Grafana dashboard JSON file you can import. + +## Sandboxing + +We've seen in some client configurations issues where Bazel in particular can leave zombie processes around on workers. +To solve this, we've two options around sandboxing that are currently switched off by default for backwards +compatibility, but are recommended for production configurations + +```json5 +workers: [{ + local: { + use_namespaces: true, + use_mount_namespace: true + // rest of your config + } +}] +``` + +`use_namespaces` enables process namespacing for workers, and `use_mount_namespace` then also isolates the worker +root in a new mount namespace. Note, `use_mount_namespace` only works if `use_namespaces` is switched on as well. + ## A reference production config A working complete production config (CAS + AC + scheduler + workers, diff --git a/web/apps/docs/content/docs/faq/rust.mdx b/web/apps/docs/content/docs/faq/rust.mdx index d68b53da1..9e2a7b6c3 100644 --- a/web/apps/docs/content/docs/faq/rust.mdx +++ b/web/apps/docs/content/docs/faq/rust.mdx @@ -3,9 +3,9 @@ title: Why Rust? description: Memory safety without garbage collection — the right shape for a service that has to be fast and never lie. --- -Three reasons: +## Three reasons -## No GC pauses +### No GC pauses Build infrastructure runs under sustained load. A scheduler hiccup during a `p99` request is the difference between "the build felt @@ -16,7 +16,7 @@ heap. Rust has no garbage collector. The tail latency on a NativeLink cluster comes from network and storage, not the language runtime. -## Memory safety +### Memory safety A miscompiled binary that ships across an organisation is a serious problem. Worse, build systems are pure-function critical @@ -27,7 +27,7 @@ Rust catches those bugs at compile time. The class of incident involving "we shipped something that wasn't what we compiled" doesn't happen. -## Throughput +### Throughput A single NativeLink instance handles over a billion build requests per month on modest hardware. That's `~400 req/s` sustained, From 217276cef05ba5cc2937676b0c874d223a184dac Mon Sep 17 00:00:00 2001 From: Alec Maliwanag Date: Tue, 21 Jul 2026 22:57:33 +0100 Subject: [PATCH 30/84] Extend the docs FAQ to cover every documentation area (#2581) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The FAQ answered conceptual questions (cost, caching, LRE, Rust) but nothing operational. Add eight question-phrased pages distilled from the rest of the docs — client wiring, configuration, store selection, production deployment, observability, troubleshooting, architecture, and contributing — each grounded in and linking back to the full page it summarizes. Also add a "Contributing to Docs or the Website" section to CONTRIBUTING.md: the old web/platform bun setup/docs/preview workflow no longer exists, and nothing documented the current Bun + Turborepo web/ workspace or that it sits outside the Bazel build. Signed-off-by: Alec Maliwanag --- .../vocabularies/TraceMachina/accept.txt | 1 + CONTRIBUTING.md | 38 ++++++++++ .../content/docs/contribute/guidelines.mdx | 2 +- .../docs/content/docs/faq/architecture.mdx | 70 +++++++++++++++++++ web/apps/docs/content/docs/faq/clients.mdx | 65 +++++++++++++++++ .../docs/content/docs/faq/configuration.mdx | 68 ++++++++++++++++++ .../docs/content/docs/faq/contributing.mdx | 59 ++++++++++++++++ web/apps/docs/content/docs/faq/deployment.mdx | 68 ++++++++++++++++++ web/apps/docs/content/docs/faq/meta.json | 10 ++- .../docs/content/docs/faq/observability.mdx | 58 +++++++++++++++ web/apps/docs/content/docs/faq/stores.mdx | 70 +++++++++++++++++++ .../docs/content/docs/faq/troubleshooting.mdx | 67 ++++++++++++++++++ .../docs/content/docs/reference/glossary.mdx | 13 ++-- 13 files changed, 582 insertions(+), 7 deletions(-) create mode 100644 web/apps/docs/content/docs/faq/architecture.mdx create mode 100644 web/apps/docs/content/docs/faq/clients.mdx create mode 100644 web/apps/docs/content/docs/faq/configuration.mdx create mode 100644 web/apps/docs/content/docs/faq/contributing.mdx create mode 100644 web/apps/docs/content/docs/faq/deployment.mdx create mode 100644 web/apps/docs/content/docs/faq/observability.mdx create mode 100644 web/apps/docs/content/docs/faq/stores.mdx create mode 100644 web/apps/docs/content/docs/faq/troubleshooting.mdx diff --git a/.github/styles/config/vocabularies/TraceMachina/accept.txt b/.github/styles/config/vocabularies/TraceMachina/accept.txt index 24af1afe0..f09b1dd1b 100644 --- a/.github/styles/config/vocabularies/TraceMachina/accept.txt +++ b/.github/styles/config/vocabularies/TraceMachina/accept.txt @@ -313,3 +313,4 @@ Merkle subtree hardlink multiplicatively +SELinux diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 8d05b1701..948fd4517 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -386,6 +386,44 @@ suggestions invoke it directly: vale somefile ``` +## Contributing to Docs or the Website + +The documentation site (docs.nativelink.com) and the marketing site +(nativelink.com) live in the `web/` directory — a Bun + Turborepo workspace +with two Next.js apps: `web/apps/docs` (the docs, built with Fumadocs) and +`web/apps/web` (the website). This replaces the old `web/platform` layout; +the `bun setup`, `bun docs`, and `bun preview` commands from that era no +longer exist. + +The `web/` workspace is **not** part of the Bazel build — it's built with +Bun only. Install Bun (https://bun.sh, or `pkgs.bun` is available in the nix +dev shell), then: + +```bash +cd web +bun install + +# Live dev server for just the docs at http://localhost:3001/docs +bun dev:docs + +# Production build + preview of the docs app +bun --filter @nativelink/docs build +bun --filter @nativelink/docs start # serves the build on port 3001 +``` + +Docs content is MDX under `web/apps/docs/content/docs/`. Every page needs +`title` and `description` frontmatter; sidebar order comes from each +section's `meta.json` (the top-level `content/docs/meta.json` orders the +sections themselves). Authoring conventions, available MDX components, and +style rules are documented on the site itself under +Contribute → [Working on documentation](https://docs.nativelink.com/contribute/docs) +— which is also the page to update if this workflow changes. + +The configuration reference pages under +`web/apps/docs/content/docs/reference/nativelink-config/` are autogenerated +from the Rust config crate — don't edit them by hand; regenerate with +`bun --filter @nativelink/docs gen:config-reference` from `web/`. + ## Creating releases To keep the release process in line with best practices for open source diff --git a/web/apps/docs/content/docs/contribute/guidelines.mdx b/web/apps/docs/content/docs/contribute/guidelines.mdx index fa799c18d..6b56d4cd8 100644 --- a/web/apps/docs/content/docs/contribute/guidelines.mdx +++ b/web/apps/docs/content/docs/contribute/guidelines.mdx @@ -63,7 +63,7 @@ We try to make the process predictable. - **First response within 2 business days.** If it's longer, ping us in the PR. -- **Two reviewers for non-trivial changes.** One for typo fixes. +- **One reviewer per change.** Others sometimes chime in. - **Squash on merge.** Every merged PR is one commit on `main`. ## License diff --git a/web/apps/docs/content/docs/faq/architecture.mdx b/web/apps/docs/content/docs/faq/architecture.mdx new file mode 100644 index 000000000..53f7ae530 --- /dev/null +++ b/web/apps/docs/content/docs/faq/architecture.mdx @@ -0,0 +1,70 @@ +--- +title: How does NativeLink work under the hood? +description: Four services glued together by the Remote Execution API — CAS, Action Cache, Scheduler, and Workers — where only the CAS holds durable state. +--- + +**NativeLink is four pieces glued together by the Remote Execution +API.** The CAS stores every blob keyed by its SHA-256 hash; the +Action Cache (AC) maps hash(Action) → ActionResult; the Scheduler +dispatches `Execute()` calls to matching workers; Workers fetch +inputs, run the command, and upload outputs. Each service +scales independently, and one binary can run all four. The full +walkthrough with diagrams is +[Explanations → Architecture](/explanations/architecture). + +## CAS versus Action Cache + +The CAS is the only service you must persist — losing it loses every +cached artifact. The AC is a smaller lookup table saying "this exact +computation has been done before"; losing it is harmless, because +clients re-execute and refill it. That asymmetry drives most +operational decisions (backups, replica counts, storage tiers). + +## What happens on a cache hit vs. a miss + +The client hashes the action — source digests, compiler version, +command line — and asks the AC. On a hit, outputs come straight from +the CAS in milliseconds and execution is skipped entirely. On a +miss, the Scheduler dispatches to a worker whose +`platform_properties` satisfy the action's requirements; the worker +runs the command (on Linux, optionally isolated with kernel +namespaces via the `use_namespaces` and `use_mount_namespace` +worker options), uploads outputs, and the ActionResult lands in +the AC for next time. + +## Why "stateless" schedulers and workers matter + +Schedulers hold no durable state — run several behind a load +balancer and restart them at will. Workers are disposable — scaling +them out is the cheapest way to make builds faster, which is why +autoscaling targets workers and nothing else. + +## The vocabulary that makes the docs make sense + +- **Digest** — a content hash plus a size; used everywhere instead + of file paths, which is what makes content-addressing work. +- **Instance name** — a namespace within a cluster; hashes from + different instance names never collide, so one cluster can serve + isolated teams. +- **Platform properties** — key/value tags matched between actions + ("needs Linux, GPU") and workers ("has Linux, GPU") to decide + dispatch. + +The full list is in the [glossary](/reference/glossary). + +## Does this actually pay off in practice + +Yes — see the +[real-world example](/explanations/architecture#a-real-world-example) +of LLVM contributors cutting full clang builds from 17 minutes to 4. +Adoption history and production users are in +[History](/explanations/history). + +## Further reading + +- [What is remote execution?](/faq/remote-execution) — the concept, + from the client's point of view. +- [What is remote caching?](/faq/caching) — how the hash is + computed. +- [Why Rust?](/faq/rust) — the language choice behind the no-GC + latency profile. diff --git a/web/apps/docs/content/docs/faq/clients.mdx b/web/apps/docs/content/docs/faq/clients.mdx new file mode 100644 index 000000000..33d2c2060 --- /dev/null +++ b/web/apps/docs/content/docs/faq/clients.mdx @@ -0,0 +1,65 @@ +--- +title: Which build tools work with NativeLink? +description: Any client that speaks the Remote Execution API — Bazel, Buck2, Siso, Pants, BuildStream, Goma, and CMake via recc — with no build-file rewrites. +--- + +**Any build tool that implements the Remote Execution API can point +at NativeLink without rewriting build files.** That includes Bazel, +Buck2, Siso (Chromium's Ninja replacement), Pants, BuildStream, Goma, +and plain CMake projects via [`recc`](/getting-started/other-build-systems/cmake-recc). +The full guides live in +[Getting started → Setup](/getting-started/setup) and +[Other build systems](/getting-started/other-build-systems); this page +is the short version. + +## How each client is wired + +- **Bazel** — `.bazelrc` flags: + `build --remote_cache=grpc://localhost:50051`, + `--remote_executor=...`, and `--remote_instance_name=main`. +- **Buck2** — a `[buck2_re_client]` section in `.buckconfig` with + `engine_address`, `action_cache_address`, and `cas_address` all + pointing at the same listener, plus an execution platform with + `remote_enabled = True`. +- **Siso** — environment variables: `SISO_REAPI_ADDRESS`, + `SISO_REAPI_INSTANCE`, and `RBE_service_no_security=true` for + local, non-TLS development. +- **Pants** — `remote_cache_read`, `remote_cache_write`, + `remote_store_address`, and `remote_instance_name` under + `[GLOBAL]` in `pants.toml`. +- **BuildStream** — `artifacts.servers` (with `push: true`) and the + `remote-execution` service URLs. +- **CMake** — set `recc` as the compiler launcher + (`CMAKE_C_COMPILER_LAUNCHER`) in cache-only mode; compiles happen + locally and outputs travel through NativeLink. + +All clients targeting the same cache must agree on the instance name +— see [Why isn't my build hitting the cache?](/faq/troubleshooting) +for what happens when they don't. + +## What platforms can I run the server on + +- **Linux (x86_64 and ARM64)** — Docker image + (`ghcr.io/tracemachina/nativelink`, multi-arch) or Nix. +- **macOS (Apple Silicon and Intel)** — the Docker image runs + natively on both architectures; the Nix path builds from source + (needs 15-20 GB of free disk) for a native binary. +- **Windows (x86_64 and ARM64)** — via Docker, or WSL2 following the + Linux instructions. + +## How do I confirm it's actually working + +Rebuild without changing source. With Bazel, +`--execution_log_json_file=/tmp/exec.json` lets you count hits with +`grep -c '"remoteCacheHit": true'`; unchanged targets report +`(remote cache hit)` in the console. `recc` prints an +`Action Cache hit` line at `RECC_LOG_LEVEL=info`. + +## Further reading + +- [Setup](/getting-started/setup) — running a server in under ten + minutes. +- [Classic RBE examples](/rbe/examples) — cache-only vs. full remote + execution vs. hybrid, and which to start with. +- [What is remote caching?](/faq/caching) — how the cache hash is + computed. diff --git a/web/apps/docs/content/docs/faq/configuration.mdx b/web/apps/docs/content/docs/faq/configuration.mdx new file mode 100644 index 000000000..a43ff40f6 --- /dev/null +++ b/web/apps/docs/content/docs/faq/configuration.mdx @@ -0,0 +1,68 @@ +--- +title: How do I configure NativeLink? +description: JSON5 files with four top-level sections — stores, servers, schedulers, and workers. Start cache-only and add execution later. +--- + +**NativeLink is configured with JSON5 files.** A single file per +cluster is the simplest way to start, but production deployments +usually split into one config per service so the CAS, scheduler, and +workers can scale independently — the deployment examples show both +shapes. JSON5 specifically, because it supports comments, trailing +commas, and single quotes — so operational notes live next to the +config instead of drifting in a separate wiki. The conceptual walkthrough is +[Configuration → Introduction](/configuration/intro); every knob and +default is in the autogenerated +[configuration reference](/reference/nativelink-config). + +## The four top-level sections + +- **`stores`** — named storage backends for CAS and Action Cache + data. See [Which store backend should I use?](/faq/stores) +- **`servers`** — gRPC/HTTP listeners exposing RE-API services. A + cache-only node enables `cas`, `ac`, `bytestream`, and + `capabilities`; an executor node adds `execution` and + `worker_api`. +- **`schedulers`** — optional; required only if the node accepts + `Execute()` calls. Matches actions to workers via + `supported_platform_properties`. +- **`workers`** — optional; required only if the node runs actions. + Each worker names its scheduler (`worker_api_endpoint`), its store + (`cas_fast_slow_store`), and the `platform_properties` describing + what it can run. + +## What's the simplest working config + +An in-memory CAS + AC pair behind one listener — the +`basic_cas.json5` example in the repo's +`nativelink-config/examples/`. Everything lives in RAM and vanishes +on restart: fine for a demo or a short-lived CI runner, disastrous +as a team cache. To survive restarts, switch the stores from +`memory` to `filesystem` with a `content_path`, `temp_path`, and an +`eviction_policy.max_bytes` +([Basic configurations](/configuration/basic)). + +## How do I add remote execution to a cache-only config + +Three additions: a `schedulers` block (a `simple` scheduler with +`supported_platform_properties`), `execution` + `worker_api` entries +in the server's `services`, and a `workers` array entry pointing at +the scheduler. The full diff is shown in +[Basic configurations](/configuration/basic). + +## What does a production config look like + +A sharded two-tier CAS (Redis in front of S3 via `fast_slow`, split +with a `shard` store), a small durable Action Cache, 2-3 stateless +schedulers behind a load balancer, mTLS between components, and +OTLP metrics enabled. A complete working example ships in the repo +as `nativelink-config/examples/production.json5` — start there and +trim ([Production configurations](/configuration/production)). + +## Further reading + +- [Content-defined chunking](/configuration/chunking) — cut transfer + bytes 80-90% for incrementally-changing artifacts. +- [Remote cache compression](/configuration/compression) — zstd on + the wire, enabled with one capabilities flag. +- [What are toolchains?](/faq/toolchains) — the concepts behind + `platform_properties`. diff --git a/web/apps/docs/content/docs/faq/contributing.mdx b/web/apps/docs/content/docs/faq/contributing.mdx new file mode 100644 index 000000000..7199637b9 --- /dev/null +++ b/web/apps/docs/content/docs/faq/contributing.mdx @@ -0,0 +1,59 @@ +--- +title: How do I contribute to NativeLink? +description: Open an issue first, develop with Bazel, Cargo, or Nix, sign every commit with DCO, and expect a first review response within two business days. +--- + +**Open an issue before anything bigger than a typo fix, then pick +whichever dev loop fits your change.** The complete rules live in +[Contribution guidelines](/contribute/guidelines); this is the +short version. + +## Which dev workflow should I use + +- **Cargo** ([guide](/contribute/cargo)) — fastest into a small, + localized change. `cargo build --workspace`, `cargo test -p + `. The Rust toolchain is pinned in `rust-toolchain.toml`; + `protoc` 3+ must be on PATH. +- **Bazel** ([guide](/contribute/bazel)) — what CI runs. + `bazel test //...` (first build 10-20 minutes, seconds after). + Editor integration via `bazel run //tools:gen-rust-project`. +- **Nix** ([guide](/contribute/nix)) — `nix develop` reproduces the + exact CI toolchain (Cargo, Bazel, protoc, mold, clang, linters). + Required for LRE work; needs 15-20 GB free disk on first run. + +A green `cargo test --workspace` does not guarantee green CI — some +crates are Bazel-only, and CI runs Bazel. Run `bazel test //...` +before submitting if you touched anything Bazel-specific. + +## What the review process expects + +- **DCO sign-off on every commit** (`git commit -s`) — this is the + formal license declaration for your contribution. +- **Formatter and linter clean**: `cargo fmt --all`, + `cargo clippy --all-targets`. +- **One change per PR**, imperative-mood title, a body that explains + *why*, and tests for new behavior. +- First response within 2 business days (ping the PR if slower); one + reviewer per change (others sometimes chime in); every merge is + squashed to one commit on `main`. + +Accepted: bug fixes with regression tests, performance work with +reproducible benchmarks, new storage backends/schedulers/worker +types (issue first), docs, and tooling. Rejected: RE-API breaking +changes, single-vendor code paths, and refactoring for its own sake. + +## How do I contribute to the docs or website + +The docs live in `web/apps/docs/content/docs/` as MDX; +`cd web && bun install && bun dev:docs` serves them at +localhost:3001/docs with live reload. Frontmatter needs `title` and +`description`; sidebar order comes from each section's `meta.json`. +Style rules and available components are in +[Working on documentation](/contribute/docs). + +## Further reading + +- [Contribution guidelines](/contribute/guidelines) — the full + accepted/rejected list and licensing details. +- [Is NativeLink free?](/faq/cost) — how the FSL + BSL module + licensing fits together, including contributor waivers. diff --git a/web/apps/docs/content/docs/faq/deployment.mdx b/web/apps/docs/content/docs/faq/deployment.mdx new file mode 100644 index 000000000..7a4a877d4 --- /dev/null +++ b/web/apps/docs/content/docs/faq/deployment.mdx @@ -0,0 +1,68 @@ +--- +title: How do I run NativeLink in production? +description: Pick one of three reference architectures, deploy on Kubernetes or plain VMs, autoscale workers on queue depth, and size storage per developer. +--- + +**Most teams land on one of three shapes.** A single-node cache (one +machine, one binary, no remote execution — fine up to ~10 +developers), a cache cluster with autoscaling workers (Redis hot +tier + S3 durable tier behind a load balancer — the standard +production deployment for 50-500 engineers), or a multi-region +cluster with per-region storage for regulated environments. The +decision guide is [On-prem overview](/deployment/on-prem-overview); +whether to self-host at all is covered in +[NativeLink on-prem](/getting-started/on-prem). + +## What do I actually deploy + +Four services: CAS (stateful — the only thing you must persist), Action +Cache (stateful but disposable), Scheduler (stateless — run 2-3 +behind a load balancer), and Workers (stateless and disposable — +autoscaling them is the cheapest speedup). One binary serves all +four; split them into separate processes beyond single-developer +scale. See [How does NativeLink work?](/faq/architecture) + +On Kubernetes that maps to a CAS StatefulSet (3+ replicas, one PVC +each, composed into one addressable store via the `shard` backend), +a scheduler Deployment, and an autoscaled worker Deployment. A +reference Helm chart with development, staging, and production +values lives in the repo's `deployment-examples/kubernetes/` +([Kubernetes](/deployment/kubernetes)). The Ingress in front must +speak gRPC end-to-end. + +## How should workers scale + +On queue depth, not CPU. Workers spend most of their time blocked on +I/O, so CPU-based scaling reacts too slowly. The reference HPA +targets the `nativelink_scheduler_queue_depth` metric (example: +`minReplicas: 4`, `maxReplicas: 200`, target average value 20). + +## How much capacity do I need + +Per active developer: 15-20 GB CAS storage for C++, 5-10 GB +for Go/Rust. Worker CPU: ~1 vCPU per concurrent action. Scheduler +memory: ~100 MB per 1,000 in-flight actions. At least 1 Gbps between +workers and CAS — cache reads are the hot path. CAS storage +dominates cost in a healthy cluster. + +## What about backups + +Depends on the CAS backend: + +- **Filesystem** — `rsync` or ZFS/Btrfs snapshots. +- **S3-compatible** — versioning, lifecycle policies, and + cross-region replication for DR. +- **Redis** — treat as ephemeral; loss is a cache miss. + +The Action Cache can be wiped without data loss. + +## Further reading + +- [Production configurations](/configuration/production) — the + config side: sharded stores, mTLS, HA schedulers. +- [How do I monitor NativeLink?](/faq/observability) — metrics and + alert thresholds. +- [Persistent workers](/deployment/persistent-workers) — 5-20x for + JVM-style toolchains; not worth it for plain clang. +- [Chromium](/deployment/chromium) — a worked example with expected + cache-hit rates. diff --git a/web/apps/docs/content/docs/faq/meta.json b/web/apps/docs/content/docs/faq/meta.json index 22e2d0e38..e9e56b43c 100644 --- a/web/apps/docs/content/docs/faq/meta.json +++ b/web/apps/docs/content/docs/faq/meta.json @@ -3,11 +3,19 @@ "cost", "caching", "remote-execution", + "architecture", "lre", "toolchains", "hermeticity", "nix", - "rust" + "rust", + "clients", + "configuration", + "stores", + "deployment", + "observability", + "troubleshooting", + "contributing" ], "title": "FAQ" } diff --git a/web/apps/docs/content/docs/faq/observability.mdx b/web/apps/docs/content/docs/faq/observability.mdx new file mode 100644 index 000000000..e9e74bb90 --- /dev/null +++ b/web/apps/docs/content/docs/faq/observability.mdx @@ -0,0 +1,58 @@ +--- +title: How do I monitor NativeLink? +description: NativeLink emits OTLP metrics only — route them through an OpenTelemetry Collector or Prometheus's OTLP receiver, and opt in to cache metrics with the cache_metrics store wrapper. +--- + +**NativeLink emits OTLP metrics only — there is no direct Prometheus +scrape endpoint.** Two working paths: send OTLP to an OpenTelemetry +Collector and let Prometheus scrape the Collector's exporter, or +send OTLP/HTTP straight to Prometheus with its OTLP receiver enabled +(`--web.enable-otlp-receiver`). Full wiring, dashboards, and a +Docker Compose quickstart are in +[Metrics & observability](/deployment/metrics). + +## Why am I seeing no cache metrics + +Cache metrics are opt-in: you must wrap the CAS/AC store with the +`cache_metrics` store wrapper in config. Enabling OTEL alone only +gets you execution-pipeline metrics. Without the wrapper NativeLink +builds the same store graph with zero metrics overhead. + +Note the licensing carve-out: metrics are one of the few modules +under the Business Source License — individual developer use is +fine, shared/production use needs Cloud, Enterprise, or an +(intentionally inexpensive) commercial license. See +[Is NativeLink free?](/faq/cost) + +## What should I alert on + +Starting thresholds from the reference alert rules: + +- **Error rate** above 5% for 5 minutes — check worker failures and + scheduler logs. +- **Queue backlog** above 100 queued actions for 15 minutes — add + workers, or check worker matching. +- **Cache eviction rate** high for 10 minutes — increase storage or + tune the eviction policy. + +From the production-config side, also watch +`nativelink_cas_request_duration_seconds_p99` (sub-millisecond when +healthy; >50 ms is page-worthy) and +`nativelink_worker_connected_count` (sudden drops point at the +scheduler). + +## Metrics aren't showing up at all + +Three checks, in order: `env | grep OTEL_` for the exporter +variables, the Collector's health endpoint (`:13133/health`), and +the Collector's own metrics (`:8888/metrics`, grep +`otelcol_receiver`) to confirm data is arriving. Prometheus +complaining about out-of-order samples wants a larger +`storage.tsdb.out-of-order-time-window` (the quickstart uses 30m). + +## Further reading + +- [Metrics & observability](/deployment/metrics) — every metric and + label, plus Grafana dashboards. +- [How do I run NativeLink in production?](/faq/deployment) — the + queue-depth metric doubles as the worker autoscaling signal. diff --git a/web/apps/docs/content/docs/faq/stores.mdx b/web/apps/docs/content/docs/faq/stores.mdx new file mode 100644 index 000000000..7800662ef --- /dev/null +++ b/web/apps/docs/content/docs/faq/stores.mdx @@ -0,0 +1,70 @@ +--- +title: Which store backend should I use? +description: Memory for demos, filesystem for single nodes, a cloud object store for anything multi-node — then compose wrappers like verify, compression, and fast_slow on top. +--- + +**Pick the terminal store by durability and sharing needs, then +compose wrapper stores on top.** A store is a named backend declared +once in the top-level `stores` array; its type is whichever key +appears inside the object. The prose companion to the generated +reference is [Store overview](/reference/nativelink-config/store-overview). + +## Terminal stores (hold actual bytes) + +- **`memory`** — fastest, wiped on restart. Dev setups and fast + tiers. +- **`filesystem`** — survives restarts, single-node. Small-team + caches and CI runners with persistent volumes. +- **`experimental_cloud_object_store`** — durable and shared across + nodes; one store type with six providers (`aws`, `gcs`, `azure`, + `ontap`, `r2`, `oci`). +- **`redis_store`** — shared, low-latency hot tier; pair it with + `size_partitioning` since Redis caps uploads around 256-512 MB. +- **`experimental_mongo`** — durable, shared, supports scheduler + change streams. + +## Wrapper stores (add behavior in front of another store) + +Wrappers nest arbitrarily deep. The shape most self-hosted +production clusters converge on, outermost first: + +- **`verify`** — hash + size checking, wrapping +- **`compression`** — lz4, wrapping +- **`fast_slow`** — where `fast` is a small in-memory tier and + `slow` is the durable cloud object store. + +Sharing one backend between two store trees? Use `ref_store` to +reference a store declared elsewhere by name instead of declaring it +twice. + +## The gotchas worth knowing up front + +- **`fast_slow` doesn't guarantee durability.** It never checks that + an object in `fast` also exists in `slow`. For artifacts that must + survive a fast-tier wipe (remote execution outputs), write through + both tiers deliberately. +- **`dedup` goes inside `compression`, not the other way around.** + Compressing before deduplication makes content-identical chunks + compress slightly differently, and deduplication stops working. +- **Some wrappers are CAS-only or AC-only.** `existence_cache` and + `size_partitioning` are CAS-only; `completeness_checking` is + AC-only. Misplacing them isn't a config error — it's a confusing + correctness bug. + +## How big should each piece be + +Rules of thumb from [On-prem overview](/deployment/on-prem-overview): +5-20 GB of CAS storage per active developer (C++ high, Go +low), Redis sized for the last 24-48 hours of artifacts (~20-40 GiB +for 1 TB/day of traffic), and an Action Cache typically under 1% of +CAS by bytes — losing it is harmless but forces rebuilds until it +warms back up. + +## Further reading + +- [Production configurations](/configuration/production) — the full + sharded S3 + Redis composition. +- [Basic configurations](/configuration/basic) — filesystem and + compression starters. +- [Store overview](/reference/nativelink-config/store-overview) — + every terminal and wrapper store, with JSON5 snippets. diff --git a/web/apps/docs/content/docs/faq/troubleshooting.mdx b/web/apps/docs/content/docs/faq/troubleshooting.mdx new file mode 100644 index 000000000..b66f48ede --- /dev/null +++ b/web/apps/docs/content/docs/faq/troubleshooting.mdx @@ -0,0 +1,67 @@ +--- +title: Why isn't my build hitting the cache? +description: Almost always an instance-name or toolchain mismatch between the client that wrote and the client that reads — plus a handful of known silent failure modes. +--- + +**Cache writes succeed but reads always miss? Compare instance names +and toolchains first.** Action hashes computed under different +`instance_name` values, or by differently-configured toolchains, +never collide — so the cache looks empty even though it's full. This +isn't an error anywhere; it's silent. + +## The usual suspects, in order + +- **Instance-name mismatch.** Every client targeting a cache must + use the same instance name (`--remote_instance_name` for Bazel). + A server config that omits `instance_name` defaults to `""` — the + same as Bazel's default — but if the server sets one, a client + that doesn't send it fails with + `'instance_name' not configured for ''`. +- **Toolchain drift.** A cache entry written by one compiler version + is invisible to another. Chromium builds hit this as + "writes succeed, reads always miss." See + [What are toolchains?](/faq/toolchains) +- **Only half the flags set.** Bazel needs `--remote_cache` even + when `--remote_executor` is set (they can be the same address) — + otherwise `Execute` is called with a digest that was never + uploaded and fails with `Action ... is missing from CAS`. Pants + similarly needs both `remote_cache_read` and `remote_cache_write`. + +## My actions queue forever and nothing happens + +The scheduler's `supported_platform_properties` must be satisfiable +by at least one worker's `platform_properties`, or matching actions +sit queued indefinitely with no error surfaced. Actions falling back +to *local* execution are the same disease in different clients: +Buck2 with `remote_enabled = False` on the selected platform, Siso +requesting properties no worker advertises. + +## Known sharp edges + +- **Docker on SELinux-enforcing distributions** (Fedora, RHEL): the mounted config + needs an explicit `:Z` label or you get "Permission denied on + /config". +- **LRE: `nix develop` did nothing.** The flake module checks for a + `.git` directory and silently skips installation without one — + `git init && git add -A` first + ([Local Remote Execution](/rbe/local-remote-execution)). +- **LRE: `Unable to resolve host TODO`.** The generated + `user.bazelrc` ships with literal `TODO` placeholders for the + cache/executor endpoints; the error actually confirms everything + else worked — fill them in. +- **Bazel 9.1.0 + chunking + `--disk_cache`** corrupts outputs; use + 9.1.1+ ([Content-defined chunking](/configuration/chunking)). + +## How do I see what's actually happening + +Bazel: `--execution_log_json_file=/tmp/exec.json`, then grep for +`remoteCacheHit`. Siso: the siso log shows the platform properties +each action requested. Cluster-side: +[How do I monitor NativeLink?](/faq/observability) + +## Further reading + +- [Setup → Troubleshooting](/getting-started/setup) — per-client + wiring checks. +- [Local cache and executor](/rbe/local-testing) — a known-good + single-machine setup to test against. diff --git a/web/apps/docs/content/docs/reference/glossary.mdx b/web/apps/docs/content/docs/reference/glossary.mdx index aba4d81e7..cfdb455c2 100644 --- a/web/apps/docs/content/docs/reference/glossary.mdx +++ b/web/apps/docs/content/docs/reference/glossary.mdx @@ -67,13 +67,16 @@ in-flight actions. ## Worker The process that runs an action. Fetches inputs from CAS, runs the -command in a sandbox, uploads outputs to CAS. +command, uploads outputs to CAS. -## Sandbox +## Isolation -The isolated environment a worker runs an action in. On Linux, -typically `bwrap` or `landlock`; on macOS, `sandbox-exec`. The -sandbox makes hermeticity enforceable. +Keeping an action from seeing state outside its declared inputs. +On Linux, workers can isolate actions with kernel namespaces via +the `use_namespaces` and `use_mount_namespace` worker options, +which also reap zombie processes and improve hermeticity. +NativeLink doesn't currently integrate external sandboxing tools +like `bwrap`, `landlock`, or `sandbox-exec`. ## Toolchain From 2cbf21ef52f78ea95d0eb11f9f4396022619b70d Mon Sep 17 00:00:00 2001 From: Ernesto Cambuston Date: Tue, 21 Jul 2026 17:12:48 -0700 Subject: [PATCH 31/84] Prefetch directory-cache tree protos with one GetTree stream (#2546) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit On a directory-cache miss, the worker fetches Directory protos level by level: each level's protos are only discoverable from its parents, so a cold tree pays one worker->CAS round trip per tree DEPTH before file fetching can even start. Real input trees are commonly 10+ levels deep, so at millisecond RTTs this serialization alone costs tens to hundreds of milliseconds per cold action, and GetTree exists in REAPI precisely to avoid it. Add opt-in DirectoryCacheConfig.experimental_get_tree_prefetch: on a cache miss, issue a single GetTree stream (one round trip, one fetch permit) and key the returned protos by their re-computed digest (the stream carries protos, not digests; hashing uses the caller's digest function). construct_directory consults the map and falls back to the existing per-level fetch for any missing proto, any stream failure, or when the slow tier is not a grpc store — so behavior can only degrade to today's path, never fail because of the prefetch. Measured RPC-shape A/B (real CasServer over TCP, client-injected RTT, identical proto counts): depth-32 chain at 25ms RTT: 877ms -> 26ms (33.7x); wide 221-proto depth-3 tree at 25ms RTT: 164ms -> 30ms (5.5x). End-to-end validated against a real server: a 4,400-file / 221-proto tree constructs entirely from one prefetched stream (zero per-level fetches) with byte-identical output. Off by default: the serving CAS walks the tree against its own backing store to answer GetTree, so deployments should enable this only when directory protos are served from a fast tier. Claude-Session: https://claude.ai/code/session_01UXVtatcR9YMecBiu9RjwpC Co-authored-by: Claude Fable 5 Co-authored-by: Marcus Eagan --- .../vocabularies/TraceMachina/accept.txt | 2 + nativelink-config/src/cas_server.rs | 19 ++ nativelink-worker/src/directory_cache.rs | 165 ++++++++++++- nativelink-worker/src/local_worker.rs | 1 + .../tests/directory_cache_test.rs | 218 +++++++++++++++++- 5 files changed, 393 insertions(+), 12 deletions(-) diff --git a/.github/styles/config/vocabularies/TraceMachina/accept.txt b/.github/styles/config/vocabularies/TraceMachina/accept.txt index f09b1dd1b..b3da2fbc2 100644 --- a/.github/styles/config/vocabularies/TraceMachina/accept.txt +++ b/.github/styles/config/vocabularies/TraceMachina/accept.txt @@ -296,6 +296,8 @@ shellexpand pluggable [Cc]allout hostnames +prefetch +protos Serde camelCase SDK diff --git a/nativelink-config/src/cas_server.rs b/nativelink-config/src/cas_server.rs index 07a3d233b..708c8a923 100644 --- a/nativelink-config/src/cas_server.rs +++ b/nativelink-config/src/cas_server.rs @@ -1090,6 +1090,25 @@ pub struct DirectoryCacheConfig { /// Default: 64 #[serde(default = "default_directory_cache_max_concurrent_fetches")] pub max_concurrent_fetches: usize, + /// On a directory-cache miss, prefetch every `Directory` proto of the + /// tree with one logical `GetTree` traversal instead of fetching protos + /// level by level (which costs one round trip per tree DEPTH). The + /// [REAPI request contract] permits servers to impose their own limit even + /// when no page size is specified, and the [REAPI response contract] + /// requires clients to continue with the returned page token. A paginated + /// traversal may therefore use multiple RPCs. Only takes effect when the + /// slow tier is a `grpc` store; + /// any prefetch failure falls back to the per-level path. Worthwhile when + /// worker-to-CAS latency is non-trivial and trees are deep; measured 5-34x + /// on the proto phase at 5-25ms RTT. Note the serving CAS pays the tree walk + /// against its own backend, so its directory protos should be served + /// from a fast tier. + /// + /// [REAPI request contract]: https://github.com/bazelbuild/remote-apis/blob/becdd8f9ff811df88a22d3eadd6341753d51d167/build/bazel/remote/execution/v2/remote_execution.proto#L1932-L1942 + /// [REAPI response contract]: https://github.com/bazelbuild/remote-apis/blob/becdd8f9ff811df88a22d3eadd6341753d51d167/build/bazel/remote/execution/v2/remote_execution.proto#L1961-L1965 + /// Default: false + #[serde(default)] + pub experimental_get_tree_prefetch: bool, } const fn default_directory_cache_max_entries() -> usize { diff --git a/nativelink-worker/src/directory_cache.rs b/nativelink-worker/src/directory_cache.rs index cd8d609e4..544c9e256 100644 --- a/nativelink-worker/src/directory_cache.rs +++ b/nativelink-worker/src/directory_cache.rs @@ -28,16 +28,19 @@ use nativelink_metric::{ MetricFieldData, MetricKind, MetricPublishKnownKindData, MetricsComponent, group, publish, }; use nativelink_proto::build::bazel::remote::execution::v2::{ - Directory as ProtoDirectory, DirectoryNode, FileNode, SymlinkNode, + Directory as ProtoDirectory, DirectoryNode, FileNode, GetTreeRequest, SymlinkNode, }; use nativelink_store::ac_utils::get_and_decode_digest; use nativelink_store::cas_utils::is_zero_digest; use nativelink_store::fast_slow_store::FastSlowStore; use nativelink_store::filesystem_store::{FileEntry, FilesystemStore}; +use nativelink_store::grpc_store::GrpcStore; use nativelink_util::background_spawn; use nativelink_util::common::DigestInfo; +use nativelink_util::digest_hasher::{DigestHasher, DigestHasherFunc, default_digest_hasher_func}; use nativelink_util::fs_util::{CloneMethod, hardlink_directory_tree, set_dir_writable_recursive}; use nativelink_util::store_trait::{StoreKey, StoreLike}; +use prost::Message; use tokio::fs; /// Maximum number of concurrently-polled node materializations (file @@ -101,6 +104,8 @@ pub struct DirectoryCacheConfig { /// see the config-crate doc for the `experimental_read_batching` /// interaction. Must be > 0. Default: 64. pub max_concurrent_fetches: usize, + /// See `nativelink-config`'s `DirectoryCacheConfig::experimental_get_tree_prefetch`. + pub experimental_get_tree_prefetch: bool, } impl Default for DirectoryCacheConfig { @@ -111,6 +116,7 @@ impl Default for DirectoryCacheConfig { cache_root: std::env::temp_dir().join("nativelink_directory_cache"), experimental_subtree_caching: false, max_concurrent_fetches: DEFAULT_MAX_CONCURRENT_FETCHES, + experimental_get_tree_prefetch: false, } } } @@ -399,7 +405,9 @@ impl DirectoryCache { /// * `Ok(false)` - Cache miss (directory was constructed) /// * `Err` - Error during construction or hardlinking pub async fn get_or_create(&self, digest: DigestInfo, dest_path: &Path) -> Result { - let (hit, _size) = self.get_or_create_entry(digest, dest_path).await?; + let (hit, _size) = self + .get_or_create_entry(digest, dest_path, None, true) + .await?; Ok(hit) } @@ -409,10 +417,19 @@ impl DirectoryCache { /// `(hit, size)` where `hit` is true when the destination was served by /// hardlinking an existing cache entry, and `size` is the entry's /// recorded size in bytes. + /// + /// `protos` is a map of prefetched `Directory` protos consulted during + /// construction (see [`Self::prefetch_tree_protos`]); `prefetch_on_miss` + /// is true only for root-level calls, which issue the tree's logical + /// `GetTree` prefetch on a cold miss. Subtree flights pass the root's + /// map down instead, so cache hits (root or subtree) never prefetch. A + /// server may paginate that traversal across multiple RPCs. async fn get_or_create_entry( &self, digest: DigestInfo, dest_path: &Path, + protos: Option<&HashMap>, + prefetch_on_miss: bool, ) -> Result<(bool, u64), Error> { self.maybe_log_summary(); @@ -446,7 +463,9 @@ impl DirectoryCache { // outcome so it cannot grow unbounded. The guard (`_guard`) is still // held until the end of this function — `forget_construction_lock` // only unmaps the Arc; any waiter already cloned it before blocking. - let result = self.construct_and_materialize(digest, dest_path).await; + let result = self + .construct_and_materialize(digest, dest_path, protos, prefetch_on_miss) + .await; self.forget_construction_lock(&digest).await; result } @@ -499,10 +518,14 @@ impl DirectoryCache { /// The cache-miss body, run while holding the per-digest construction /// guard. Split out so `get_or_create_entry` can unconditionally clean up /// the construction-lock map entry afterwards on every exit path. + /// `protos` and `prefetch_on_miss` are documented on + /// [`Self::get_or_create_entry`]. async fn construct_and_materialize( &self, digest: DigestInfo, dest_path: &Path, + protos: Option<&HashMap>, + prefetch_on_miss: bool, ) -> Result<(bool, u64), Error> { // Re-check: another task may have just constructed this digest while // we waited on the construction lock. If the entry turns out to be @@ -532,9 +555,19 @@ impl DirectoryCache { // with the CAS and every other in-flight action that hardlinked the // same blob — the inode-corruption bug PR #2347 fixed. let cache_path = self.get_cache_path(&digest); + // Root-level cold miss: prefetch the whole tree's `Directory` protos + // with one logical `GetTree` traversal, following server pagination. + // Subtree flights never prefetch — they inherit the root's map (or + // `None`) through `protos` instead. + let prefetched = if prefetch_on_miss { + Box::pin(self.prefetch_tree_protos(digest)).await + } else { + None + }; + let protos = prefetched.as_ref().or(protos); let temp_path = self.allocate_scratch_path(TEMP_PREFIX); let mut temp_guard = ScratchGuard::new(temp_path.clone()); - let size = match self.construct_directory(digest, &temp_path).await { + let size = match self.construct_directory(digest, &temp_path, protos).await { Ok(size) => size, Err(e) => { temp_guard.disarm(); @@ -739,16 +772,103 @@ impl DirectoryCache { /// /// Each directory's final mode (0o755) is set at creation time, so no /// separate recursive permission pass is needed after construction. + /// Prefetches every `Directory` proto of `root`'s tree with a logical + /// `GetTree` traversal, following `next_page_token` when the server + /// paginates it, and keys each proto by its re-computed digest (the + /// responses carry protos, not digests). Returns `None` — meaning + /// "use the per-level fetch path" — when the feature is disabled, the + /// slow tier is not a `GrpcStore`, or the stream fails; a `Some` map + /// may also be incomplete, which `construct_directory` tolerates by + /// fetching any missing proto individually. One fetch permit covers + /// the entire paginated traversal. + async fn prefetch_tree_protos( + &self, + root: DigestInfo, + ) -> Option> { + if !self.config.experimental_get_tree_prefetch { + return None; + } + let grpc_store = self + .cas_store + .slow_store() + .downcast_ref::(None)?; + let digest_hasher = opentelemetry::Context::current() + .get::() + .map_or_else(default_digest_hasher_func, |v| *v); + let result: Result, Error> = async { + let _permit = self.acquire_fetch_permit().await?; + let mut protos = HashMap::new(); + let mut page_token = String::new(); + loop { + let mut stream = grpc_store + .get_tree(tonic::Request::new(GetTreeRequest { + instance_name: String::new(), + root_digest: Some(root.into()), + page_size: 0, + page_token, + digest_function: digest_hasher.proto_digest_func().into(), + })) + .await + .err_tip(|| "in prefetch_tree_protos")? + .into_inner(); + let mut next_page_token = String::new(); + while let Some(response) = stream + .message() + .await + .map_err(Error::from) + .err_tip(|| "reading GetTree stream in prefetch_tree_protos")? + { + next_page_token = response.next_page_token; + for directory in response.directories { + // GetTree yields protos without digests; recompute with + // the caller's digest function so lookups match the + // digests embedded in parent directories. + let encoded = directory.encode_to_vec(); + let mut hasher = digest_hasher.hasher(); + hasher.update(&encoded); + protos.insert(hasher.finalize_digest(), directory); + } + } + if next_page_token.is_empty() { + break; + } + page_token = next_page_token; + } + Ok(protos) + } + .await; + match result { + Ok(protos) => { + trace!(?root, protos = protos.len(), "GetTree prefetch complete"); + Some(protos) + } + Err(err) => { + debug!( + ?err, + ?root, + "GetTree prefetch failed; using per-level fetches" + ); + None + } + } + } + fn construct_directory<'a>( &'a self, digest: DigestInfo, dest_path: &'a Path, + protos: Option<&'a HashMap>, ) -> Pin> + Send + 'a>> { Box::pin(async move { debug!(?digest, ?dest_path, "Constructing directory"); - // Fetch the Directory proto (permit held only for the fetch). - let directory: ProtoDirectory = { + // Use the prefetched proto when available; otherwise fetch it + // (permit held only for the fetch). A prefetch-map miss (e.g. + // an incomplete GetTree response) degrades to the fetch path. + let prefetched = protos.and_then(|map| map.get(&digest)); + let directory: ProtoDirectory = if let Some(directory) = prefetched { + directory.clone() + } else { let _permit = self.acquire_fetch_permit().await?; get_and_decode_digest(self.cas_store.as_ref(), digest.into()) .await @@ -785,7 +905,9 @@ impl DirectoryCache { })); } for dir_node in &directory.directories { - node_futures.push(Box::pin(self.create_subdirectory(dest_path, dir_node))); + node_futures.push(Box::pin( + self.create_subdirectory(dest_path, dir_node, protos), + )); } for symlink in &directory.symlinks { node_futures.push(Box::pin(async move { @@ -988,6 +1110,7 @@ impl DirectoryCache { &self, parent: &Path, dir_node: &DirectoryNode, + protos: Option<&HashMap>, ) -> Result { let dir_path = parent.join(&dir_node.name); let digest = @@ -1006,7 +1129,9 @@ impl DirectoryCache { // byte-identical wherever else it appears; construction recurses // through this method, so nested subtrees get their own entries // too (leaf-level reuse). - let (hit, _size) = self.get_or_create_entry(digest, &dir_path).await?; + let (hit, _size) = self + .get_or_create_entry(digest, &dir_path, protos, false) + .await?; let counter = if hit { &self.subtree_hits } else { @@ -1025,7 +1150,7 @@ impl DirectoryCache { } // Recursively construct subdirectory - self.construct_directory(digest, &dir_path).await + self.construct_directory(digest, &dir_path, protos).await } /// Creates a symlink from a `SymlinkNode` @@ -2119,4 +2244,26 @@ mod tests { Ok(()) } + + #[nativelink_test] + async fn get_tree_prefetch_falls_back_without_grpc_store() -> Result<(), Error> { + // With the flag on but a non-grpc slow tier, prefetch must return + // None and construction must fall back to per-level fetches with + // identical results. + let temp_dir = TempDir::new().unwrap(); + let cache_root = temp_dir.path().join("cache"); + let (store, dir_digest) = setup_test_store(&temp_dir).await; + let config = DirectoryCacheConfig { + max_entries: 10, + max_size_bytes: 1024 * 1024, + cache_root, + experimental_get_tree_prefetch: true, + ..Default::default() + }; + let cache = DirectoryCache::new(config, store).await?; + let dest = temp_dir.path().join("dest"); + assert!(!cache.get_or_create(dir_digest, &dest).await?); + assert!(dest.join("test.txt").exists()); + Ok(()) + } } diff --git a/nativelink-worker/src/local_worker.rs b/nativelink-worker/src/local_worker.rs index a4fe64e47..e91020247 100644 --- a/nativelink-worker/src/local_worker.rs +++ b/nativelink-worker/src/local_worker.rs @@ -629,6 +629,7 @@ pub async fn new_local_worker( cache_root, experimental_subtree_caching: cache_config.experimental_subtree_caching, max_concurrent_fetches: cache_config.max_concurrent_fetches, + experimental_get_tree_prefetch: cache_config.experimental_get_tree_prefetch, }; match DirectoryCache::new(worker_cache_config, fast_slow_store.clone()).await { diff --git a/nativelink-worker/tests/directory_cache_test.rs b/nativelink-worker/tests/directory_cache_test.rs index 5848d5692..681127791 100644 --- a/nativelink-worker/tests/directory_cache_test.rs +++ b/nativelink-worker/tests/directory_cache_test.rs @@ -21,29 +21,43 @@ use std::sync::Arc; use async_trait::async_trait; use bytes::Bytes; +use futures::Stream; use nativelink_config::stores::{ - FastSlowSpec, FilesystemSpec, MemorySpec, StoreDirection, StoreSpec, + FastSlowSpec, FilesystemSpec, GrpcEndpoint, GrpcSpec, MemorySpec, Retry, StoreDirection, + StoreSpec, StoreType, }; use nativelink_error::Error; use nativelink_macro::nativelink_test; use nativelink_metric::{ MetricFieldData, MetricKind, MetricPublishKnownKindData, MetricsComponent, }; +use nativelink_proto::build::bazel::remote::execution::v2::content_addressable_storage_server::{ + ContentAddressableStorage, ContentAddressableStorageServer, +}; use nativelink_proto::build::bazel::remote::execution::v2::{ - Directory as ProtoDirectory, DirectoryNode, FileNode, SymlinkNode, + BatchReadBlobsRequest, BatchReadBlobsResponse, BatchUpdateBlobsRequest, + BatchUpdateBlobsResponse, Directory as ProtoDirectory, DirectoryNode, FileNode, + FindMissingBlobsRequest, FindMissingBlobsResponse, GetTreeRequest, GetTreeResponse, + SpliceBlobRequest, SpliceBlobResponse, SplitBlobRequest, SplitBlobResponse, SymlinkNode, }; use nativelink_store::fast_slow_store::FastSlowStore; use nativelink_store::filesystem_store::FilesystemStore; +use nativelink_store::grpc_store::GrpcStore; use nativelink_store::memory_store::MemoryStore; +use nativelink_util::background_spawn; use nativelink_util::buf_channel::{DropCloserReadHalf, DropCloserWriteHalf}; use nativelink_util::common::{DigestInfo, make_temp_path}; +use nativelink_util::digest_hasher::{DigestHasher, default_digest_hasher_func}; use nativelink_util::health_utils::{HealthStatusIndicator, default_health_status_indicator}; use nativelink_util::store_trait::{ RemoveItemCallback, Store, StoreDriver, StoreKey, StoreLike, UploadSizeInfo, }; use nativelink_worker::directory_cache::{DirectoryCache, DirectoryCacheConfig}; use prost::Message; -use tonic::Code; +use tokio::sync::Mutex; +use tonic::transport::Server; +use tonic::transport::server::TcpIncoming; +use tonic::{Code, Request, Response, Status}; use uuid::Uuid; /// Wraps a `MemoryStore` as the slow tier of a `FastSlowStore` whose fast @@ -1432,3 +1446,201 @@ async fn subtree_caching_size_accounting_not_depth_multiplied() -> Result<(), Er ); Ok(()) } + +#[derive(Clone)] +struct PaginatedGetTreeServer { + root: ProtoDirectory, + child: ProtoDirectory, + requests: Arc>>, +} + +type PaginatedGetTreeStream = + Pin> + Send + 'static>>; + +#[async_trait] +impl ContentAddressableStorage for PaginatedGetTreeServer { + type GetTreeStream = PaginatedGetTreeStream; + + async fn find_missing_blobs( + &self, + _request: Request, + ) -> Result, Status> { + Err(Status::unimplemented("not used by this test")) + } + + async fn batch_update_blobs( + &self, + _request: Request, + ) -> Result, Status> { + Err(Status::unimplemented("not used by this test")) + } + + async fn batch_read_blobs( + &self, + _request: Request, + ) -> Result, Status> { + Err(Status::unimplemented("not used by this test")) + } + + async fn get_tree( + &self, + request: Request, + ) -> Result, Status> { + const SECOND_PAGE_TOKEN: &str = "child-directory-page"; + + let request = request.into_inner(); + self.requests.lock().await.push(request.clone()); + let response = match request.page_token.as_str() { + "" => GetTreeResponse { + directories: vec![self.root.clone()], + next_page_token: SECOND_PAGE_TOKEN.to_string(), + }, + SECOND_PAGE_TOKEN => GetTreeResponse { + directories: vec![self.child.clone()], + next_page_token: String::new(), + }, + token => { + return Err(Status::invalid_argument(format!( + "unexpected token: {token}" + ))); + } + }; + Ok(Response::new(Box::pin(futures::stream::iter([Ok( + response, + )])))) + } + + async fn split_blob( + &self, + _request: Request, + ) -> Result, Status> { + Err(Status::unimplemented("not used by this test")) + } + + async fn splice_blob( + &self, + _request: Request, + ) -> Result, Status> { + Err(Status::unimplemented("not used by this test")) + } +} + +fn proto_digest(directory: &ProtoDirectory) -> DigestInfo { + let mut hasher = default_digest_hasher_func().hasher(); + hasher.update(&directory.encode_to_vec()); + hasher.finalize_digest() +} + +/// The [REAPI request contract] says a zero page size does not require a +/// server to return the entire tree in one response, and the [REAPI response +/// contract] tells the client to continue with `next_page_token`. This test +/// therefore deliberately uses the smallest tree that cannot be represented +/// by the first page alone. The first page contains only the root `Directory`, +/// whose `DirectoryNode` names and hashes the child, while the second page +/// contains only that child `Directory`. +/// +/// Using two different, content-derived directory digests matters here: the +/// public `get_or_create` operation cannot materialize the named child unless +/// the prefetch map contains both pages. The fake server also records the +/// requests so the test verifies that the continuation token was copied +/// exactly into the follow-up request and that fetching stopped as soon as the +/// server returned an empty token. No directory protos are seeded into the +/// filesystem store, and the server intentionally does not expose a +/// `ByteStream` service. Consequently, ignoring the second page cannot be +/// hidden by the normal per-level fallback: that fallback would fail while +/// fetching the absent child proto. Before the fix, the client drained the first response +/// stream, discarded the advertised child-directory page, and took precisely +/// that failing fallback path. +/// +/// [REAPI request contract]: https://github.com/bazelbuild/remote-apis/blob/becdd8f9ff811df88a22d3eadd6341753d51d167/build/bazel/remote/execution/v2/remote_execution.proto#L1932-L1942 +/// [REAPI response contract]: https://github.com/bazelbuild/remote-apis/blob/becdd8f9ff811df88a22d3eadd6341753d51d167/build/bazel/remote/execution/v2/remote_execution.proto#L1961-L1965 +#[nativelink_test] +async fn get_tree_prefetch_follows_server_pagination() -> Result<(), Error> { + let child = ProtoDirectory::default(); + let child_digest = proto_digest(&child); + let root = ProtoDirectory { + directories: vec![DirectoryNode { + name: "second-page".to_string(), + digest: Some(child_digest.into()), + }], + ..Default::default() + }; + let root_digest = proto_digest(&root); + let requests = Arc::new(Mutex::new(Vec::new())); + let service = PaginatedGetTreeServer { + root, + child, + requests: Arc::clone(&requests), + }; + let listener = TcpIncoming::bind("127.0.0.1:0".parse().unwrap()).unwrap(); + let port = listener.local_addr().unwrap().port(); + background_spawn!("paginated_get_tree_server", async move { + Server::builder() + .add_service(ContentAddressableStorageServer::new(service)) + .serve_with_incoming(listener) + .await + .unwrap(); + }); + + let grpc_spec = GrpcSpec { + instance_name: String::new(), + endpoints: vec![GrpcEndpoint { + address: format!("http://127.0.0.1:{port}"), + tls_config: None, + concurrency_limit: None, + connect_timeout_s: 0, + tcp_keepalive_s: 0, + http2_keepalive_interval_s: 0, + http2_keepalive_timeout_s: 0, + }], + store_type: StoreType::Cas, + retry: Retry::default(), + max_concurrent_requests: 0, + connections_per_endpoint: 0, + rpc_timeout_s: 1, + use_legacy_resource_names: false, + headers: HashMap::new(), + forward_headers: vec![], + experimental_read_batching: None, + }; + let fast_spec = FilesystemSpec { + content_path: make_temp_path("paginated_get_tree_cas_content"), + temp_path: make_temp_path("paginated_get_tree_cas_temp"), + ..Default::default() + }; + let fast_store: Arc = FilesystemStore::new(&fast_spec).await?; + let grpc_store = GrpcStore::new(&grpc_spec).await?; + let cas_store = FastSlowStore::new( + &FastSlowSpec { + fast: StoreSpec::Filesystem(fast_spec), + slow: StoreSpec::Grpc(grpc_spec), + fast_direction: StoreDirection::default(), + slow_direction: StoreDirection::default(), + bypass_dedup_threshold_bytes: 0, + }, + Store::new(fast_store), + Store::new(grpc_store), + ); + let cache = DirectoryCache::new( + DirectoryCacheConfig { + cache_root: make_temp_path("paginated_get_tree_directory_cache").into(), + experimental_get_tree_prefetch: true, + ..Default::default() + }, + cas_store, + ) + .await?; + + let destination = PathBuf::from(make_temp_path("paginated_get_tree_destination")); + assert!(!cache.get_or_create(root_digest, &destination).await?); + assert!(destination.join("second-page").is_dir()); + + let requests = requests.lock().await; + assert_eq!(requests.len(), 2); + assert_eq!(requests[0].page_token, ""); + assert_eq!(requests[1].page_token, "child-directory-page"); + assert_eq!(requests[0].root_digest, Some(root_digest.into())); + assert_eq!(requests[1].root_digest, Some(root_digest.into())); + + Ok(()) +} From 82a036c4a211d6009a251fd47182686f0a2bd85f Mon Sep 17 00:00:00 2001 From: Alec Maliwanag Date: Wed, 22 Jul 2026 01:17:30 +0100 Subject: [PATCH 32/84] Move the FAQ into per-page FAQ sections (#2590) The standalone /faq section answered questions away from the material they belong to. Dissolve it: every hand-written docs page now ends with a '## FAQ' section holding the questions readers actually ask about that page, with the standalone section's content redistributed to the page each answer belongs to. - Add '## FAQ' sections to all 33 hand-written pages (autogenerated config reference pages excluded), rendered as collapsible accordions (fumadocs-ui Accordions, newly registered in mdx-components.tsx): question titles visible, answers collapsed, each question deep-linkable and auto-opening from its anchor - Delete web/apps/docs/content/docs/faq/ and its sidebar entry - Redirect the 16 published /faq/* URLs to their new page#faq homes - Repoint the six internal /faq links (index, glossary, LRE) - Fix 'per active developer per week' sizing in the source pages - Update install snippets to the multi-arch v1.6.1 image and drop the Apple Silicon emulation workaround; v1.6.0+ ships arm64, earlier tags (v1.4.0, v1.0.0) were x86_64-only Signed-off-by: Alec Maliwanag Co-authored-by: Marcus Eagan --- .../docs/content/docs/configuration/basic.mdx | 21 ++++++ .../content/docs/configuration/chunking.mdx | 23 ++++++ .../docs/configuration/compression.mdx | 17 +++++ .../docs/content/docs/configuration/intro.mdx | 23 ++++++ .../content/docs/configuration/production.mdx | 24 ++++++- .../docs/content/docs/contribute/bazel.mdx | 21 +++++- .../docs/content/docs/contribute/cargo.mdx | 14 ++++ .../docs/content/docs/contribute/docs.mdx | 17 +++++ .../content/docs/contribute/guidelines.mdx | 16 +++++ web/apps/docs/content/docs/contribute/nix.mdx | 16 +++++ .../docs/content/docs/deployment/chromium.mdx | 14 ++++ .../content/docs/deployment/kubernetes.mdx | 24 ++++++- .../docs/content/docs/deployment/metrics.mdx | 24 +++++++ .../docs/deployment/oci-object-storage.mdx | 14 ++++ .../docs/deployment/on-prem-overview.mdx | 25 ++++++- .../docs/deployment/persistent-workers.mdx | 16 +++++ .../docs/explanations/architecture.mdx | 22 ++++++ .../content/docs/explanations/history.mdx | 19 +++++ .../docs/content/docs/explanations/lre.mdx | 31 +++++++- .../docs/content/docs/faq/architecture.mdx | 70 ------------------- web/apps/docs/content/docs/faq/caching.mdx | 53 -------------- web/apps/docs/content/docs/faq/clients.mdx | 65 ----------------- .../docs/content/docs/faq/configuration.mdx | 68 ------------------ .../docs/content/docs/faq/contributing.mdx | 59 ---------------- web/apps/docs/content/docs/faq/cost.mdx | 60 ---------------- web/apps/docs/content/docs/faq/deployment.mdx | 68 ------------------ .../docs/content/docs/faq/hermeticity.mdx | 68 ------------------ web/apps/docs/content/docs/faq/lre.mdx | 48 ------------- web/apps/docs/content/docs/faq/meta.json | 21 ------ web/apps/docs/content/docs/faq/nix.mdx | 49 ------------- .../docs/content/docs/faq/observability.mdx | 58 --------------- .../content/docs/faq/remote-execution.mdx | 62 ---------------- web/apps/docs/content/docs/faq/rust.mdx | 44 ------------ web/apps/docs/content/docs/faq/stores.mdx | 70 ------------------- web/apps/docs/content/docs/faq/toolchains.mdx | 52 -------------- .../docs/content/docs/faq/troubleshooting.mdx | 67 ------------------ .../content/docs/getting-started/on-prem.mdx | 34 ++++++++- .../other-build-systems/buck2.mdx | 16 +++++ .../other-build-systems/buildstream.mdx | 16 +++++ .../other-build-systems/cmake-recc.mdx | 32 +++++++-- .../other-build-systems/index.mdx | 24 +++++++ .../other-build-systems/pants.mdx | 16 +++++ .../other-build-systems/siso.mdx | 15 ++++ .../content/docs/getting-started/setup.mdx | 52 ++++++++++---- web/apps/docs/content/docs/index.mdx | 36 ++++++++-- web/apps/docs/content/docs/meta.json | 1 - web/apps/docs/content/docs/rbe/examples.mdx | 24 +++++++ .../docs/rbe/local-remote-execution.mdx | 18 +++++ .../docs/content/docs/rbe/local-testing.mdx | 16 +++++ .../docs/content/docs/rbe/nix-templates.mdx | 17 +++++ .../docs/content/docs/reference/glossary.mdx | 5 +- .../nativelink-config/store-overview.mdx | 20 ++++++ web/apps/docs/mdx-components.tsx | 3 + web/apps/docs/next.config.mjs | 24 +++++++ 54 files changed, 712 insertions(+), 1020 deletions(-) delete mode 100644 web/apps/docs/content/docs/faq/architecture.mdx delete mode 100644 web/apps/docs/content/docs/faq/caching.mdx delete mode 100644 web/apps/docs/content/docs/faq/clients.mdx delete mode 100644 web/apps/docs/content/docs/faq/configuration.mdx delete mode 100644 web/apps/docs/content/docs/faq/contributing.mdx delete mode 100644 web/apps/docs/content/docs/faq/cost.mdx delete mode 100644 web/apps/docs/content/docs/faq/deployment.mdx delete mode 100644 web/apps/docs/content/docs/faq/hermeticity.mdx delete mode 100644 web/apps/docs/content/docs/faq/lre.mdx delete mode 100644 web/apps/docs/content/docs/faq/meta.json delete mode 100644 web/apps/docs/content/docs/faq/nix.mdx delete mode 100644 web/apps/docs/content/docs/faq/observability.mdx delete mode 100644 web/apps/docs/content/docs/faq/remote-execution.mdx delete mode 100644 web/apps/docs/content/docs/faq/rust.mdx delete mode 100644 web/apps/docs/content/docs/faq/stores.mdx delete mode 100644 web/apps/docs/content/docs/faq/toolchains.mdx delete mode 100644 web/apps/docs/content/docs/faq/troubleshooting.mdx diff --git a/web/apps/docs/content/docs/configuration/basic.mdx b/web/apps/docs/content/docs/configuration/basic.mdx index 6e6e87f02..6fb823507 100644 --- a/web/apps/docs/content/docs/configuration/basic.mdx +++ b/web/apps/docs/content/docs/configuration/basic.mdx @@ -193,6 +193,27 @@ Every client targeting a given cache must use the same `instance_name`. Mismatches don't produce errors; they produce silently-empty caches. +## FAQ + + + + Each store's `eviction_policy.max_bytes` caps it; once full, the + least-recently-used entries are evicted to make room. An evicted + entry is just a future cache miss — clients rebuild and re-upload. + + + The scheduler's `supported_platform_properties` must be satisfiable + by at least one worker's `platform_properties`, or matching actions + queue indefinitely with no error surfaced. Compare those two blocks + first. + + + When more than one node needs the same cache, or when a restart + wiping state starts to hurt. That's the cue for the durable, sharded + shapes in [Production configurations](/configuration/production). + + + ## What's next - [Production configurations](/configuration/production) — the diff --git a/web/apps/docs/content/docs/configuration/chunking.mdx b/web/apps/docs/content/docs/configuration/chunking.mdx index 605284ed7..bfa28bda1 100644 --- a/web/apps/docs/content/docs/configuration/chunking.mdx +++ b/web/apps/docs/content/docs/configuration/chunking.mdx @@ -108,3 +108,26 @@ materializes the full blob, so non-chunking clients and every existing read path see ordinary blobs. Storage grows by roughly the chunk bytes for chunk-eligible blobs; pairing the CAS with a `dedup` or `compression` store composes normally. + +## FAQ + + + + No. Chunking is opt-in and advertised through the capabilities + service — clients that don't request it (or predate it) keep using + regular transfers against the same CAS. + + + Bazel 9.1.0 has a client bug that corrupts outputs when the chunking + flag is combined with `--disk_cache`. Use 9.1.1+ (or 8.7.0+). + + + They solve different transfer problems. Chunking re-transfers only + the changed parts of large, incrementally-changing blobs; + [wire compression](/configuration/compression) shrinks compressible + blobs wholesale. Compressed artifacts chunk poorly — everything after + the first changed byte re-transfers — so lean on chunking for + archives, binaries, and layers, and on compression for text-heavy + outputs. + + diff --git a/web/apps/docs/content/docs/configuration/compression.mdx b/web/apps/docs/content/docs/configuration/compression.mdx index 28389a8c3..d81e2ed93 100644 --- a/web/apps/docs/content/docs/configuration/compression.mdx +++ b/web/apps/docs/content/docs/configuration/compression.mdx @@ -88,3 +88,20 @@ Wire compression doesn't change what's stored: the CAS holds raw bytes. To also shrink bytes at rest, wrap the backing store in a [`compression` store](/reference/nativelink-config/main#compressionspec); the two compose, since one covers the wire and the other covers storage. + +## FAQ + + + + Yes. Clients only compress when the server advertises support, so + anything that doesn't request `compressed-blobs/zstd` keeps using + identity transfers against the same instance. + + + Yes, but order matters at the store level: deduplication must operate + on raw content — identical chunks stop being identical after + compression, so a `dedup` store fed compressed bytes finds no + duplicates. Wire compression is fine either way; it's undone at the + gRPC boundary before any store sees the bytes. + + diff --git a/web/apps/docs/content/docs/configuration/intro.mdx b/web/apps/docs/content/docs/configuration/intro.mdx index 75fde30dd..058d2c60c 100644 --- a/web/apps/docs/content/docs/configuration/intro.mdx +++ b/web/apps/docs/content/docs/configuration/intro.mdx @@ -160,6 +160,29 @@ autogenerated from the Rust source — every knob, every default, every constraint. Use this page for understanding, the reference for lookups. +## FAQ + + + + One file per cluster is the simplest way to start, and everything on + this page assumes it. Production deployments usually split into one + config per service — separate CAS, scheduler, and worker files — so + each service can be scaled, restarted, and deployed independently. + + + An in-memory CAS + AC behind one listener — + [`basic_cas.json5`](https://github.com/TraceMachina/nativelink/blob/main/nativelink-config/examples/basic_cas.json5) + in the repo. Fine for a demo, gone on restart; see + [Basic configurations](/configuration/basic) for shapes that survive + a reboot. + + + No — both sections are optional and only required on nodes that + accept `Execute()` calls or run actions. A cache-only node is just + `stores` and `servers`. + + + ## What's next - [Basic configurations](/configuration/basic) — runnable shapes diff --git a/web/apps/docs/content/docs/configuration/production.mdx b/web/apps/docs/content/docs/configuration/production.mdx index baae5e22d..783b7be70 100644 --- a/web/apps/docs/content/docs/configuration/production.mdx +++ b/web/apps/docs/content/docs/configuration/production.mdx @@ -210,7 +210,6 @@ What's worth alerting on: The [Metrics deployment guide](/deployment/metrics) has a Grafana dashboard JSON file you can import. - ## Sandboxing We've seen in some client configurations issues where Bazel in particular can leave zombie processes around on workers. @@ -230,6 +229,29 @@ workers: [{ `use_namespaces` enables process namespacing for workers, and `use_mount_namespace` then also isolates the worker root in a new mount namespace. Note, `use_mount_namespace` only works if `use_namespaces` is switched on as well. +## FAQ + + + + The queue lives in scheduler memory and is restored from the Action + Cache on restart — which is why running two or three stateless + schedulers behind a load balancer is enough for HA. + + + Tag the workers (`gpu: { values: ["nvidia-a100"] }`) and declare the + property as `"exact"` in the scheduler's + `supported_platform_properties`. An action requesting that value is + only ever dispatched to matching workers. + + + No — it never verifies that an object in `fast` also exists in + `slow`. For artifacts that must survive a fast-tier wipe — remote + execution outputs especially — make sure the write path covers both + tiers deliberately. More store gotchas: + [Store overview](/reference/nativelink-config/store-overview#faq). + + + ## A reference production config A working complete production config (CAS + AC + scheduler + workers, diff --git a/web/apps/docs/content/docs/contribute/bazel.mdx b/web/apps/docs/content/docs/contribute/bazel.mdx index f8b9624c2..59c1eb42b 100644 --- a/web/apps/docs/content/docs/contribute/bazel.mdx +++ b/web/apps/docs/content/docs/contribute/bazel.mdx @@ -76,6 +76,21 @@ editor of choice will pick up. reuse Bazel's local cache; pointing at a remote NativeLink cluster makes a fresh clone fast for everyone. -For non-Bazel workflows, see -[Develop with Cargo](/contribute/cargo) or -[Develop with Nix](/contribute/nix). +## FAQ + + + + It bootstraps the toolchain and builds every dependency from source. + Subsequent builds are seconds — and pointing `--remote_cache` at a + NativeLink cluster (above) makes even a fresh clone fast. + + + No — target what you touched + (`bazel test //nativelink-store/tests:s3_store_test`), then run the + full `bazel test //...` once before submitting. + + For non-Bazel workflows, see + [Develop with Cargo](/contribute/cargo) or + [Develop with Nix](/contribute/nix). + + diff --git a/web/apps/docs/content/docs/contribute/cargo.mdx b/web/apps/docs/content/docs/contribute/cargo.mdx index a50a2053c..611846937 100644 --- a/web/apps/docs/content/docs/contribute/cargo.mdx +++ b/web/apps/docs/content/docs/contribute/cargo.mdx @@ -81,3 +81,17 @@ A few flags that help: For the Bazel-based workflow, see [Develop with Bazel](/contribute/bazel). + +## FAQ + + + + Not for most crates — the whole workspace builds with plain Cargo. + You need Bazel for the Bazel-only crates and for a pre-submit + `bazel test //...`, since that's what CI runs. + + + The gRPC services are generated from protobuf definitions at build + time; without `protoc` 3+ on `$PATH`, those generation steps fail. + + diff --git a/web/apps/docs/content/docs/contribute/docs.mdx b/web/apps/docs/content/docs/contribute/docs.mdx index 2f1164a38..c2788831e 100644 --- a/web/apps/docs/content/docs/contribute/docs.mdx +++ b/web/apps/docs/content/docs/contribute/docs.mdx @@ -101,3 +101,20 @@ For tabbed content, use Fumadocs's `Tabs` and `Tab`: Same flow as any other change — see [Contribution guidelines](/contribute/guidelines). Small docs PRs land same-day in most cases. + +## FAQ + + + + At the bottom of the page they belong to — each page ends with a + `## FAQ` section holding the questions readers actually ask about + that material. Add new Q&As there rather than to a standalone FAQ + section; there isn't one. + + + The same pre-commit suite as any PR, including + [Vale](https://vale.sh) style linting and typo checks over the MDX. + Run `pre-commit run --files ` locally to catch + failures before CI does. + + diff --git a/web/apps/docs/content/docs/contribute/guidelines.mdx b/web/apps/docs/content/docs/contribute/guidelines.mdx index 6b56d4cd8..a55605077 100644 --- a/web/apps/docs/content/docs/contribute/guidelines.mdx +++ b/web/apps/docs/content/docs/contribute/guidelines.mdx @@ -79,6 +79,22 @@ Meaningful contributors may be eligible for license waivers for Business Source License modules. Open an issue or contact the maintainers before depending on a waiver. +## FAQ + + + + [Cargo](/contribute/cargo) for small, localized changes — it's the + fastest way in. [Bazel](/contribute/bazel) when you want exactly what + CI runs. [Nix](/contribute/nix) for the fully-pinned shell, and + required for LRE work. + + + CI runs Bazel, and some crates are Bazel-only (the LRE flake outputs, + the Bazel-generated proto bindings). Run `bazel test //...` before + submitting if you touched anything Bazel-specific. + + + ## What's next Pick the dev environment that fits how you work: diff --git a/web/apps/docs/content/docs/contribute/nix.mdx b/web/apps/docs/content/docs/contribute/nix.mdx index db332cb25..3a84fc4d7 100644 --- a/web/apps/docs/content/docs/contribute/nix.mdx +++ b/web/apps/docs/content/docs/contribute/nix.mdx @@ -82,3 +82,19 @@ If you're making a small, localised change — typo fix, doc tweak, a unit test — the Cargo workflow is faster to get into. Nix shines for work touching the build graph, the LRE flow, or anything that needs exact toolchain reproducibility. + +## FAQ + + + + A package manager that builds every package in isolation and + addresses the result by a hash of all its inputs — which is why two + machines running `nix develop` on this repo get bit-identical + toolchains. + + + Only for [LRE](/explanations/lre) work. For everything else it's the + most reproducible option, not the mandatory one — the Cargo and Bazel + workflows stand on their own. + + diff --git a/web/apps/docs/content/docs/deployment/chromium.mdx b/web/apps/docs/content/docs/deployment/chromium.mdx index 30bfd406f..6c1a0a75a 100644 --- a/web/apps/docs/content/docs/deployment/chromium.mdx +++ b/web/apps/docs/content/docs/deployment/chromium.mdx @@ -99,6 +99,20 @@ directory in the source tree has the worker config Samsung uses. toolchain version mismatch between the action that wrote the cache and the one trying to read. +## FAQ + + + + No — pointing Siso at NativeLink is configuration only: GN args plus + the Siso environment variables. The same applies to Chromium forks; + Samsung Internet runs this pattern in production. + + + Yes. A cache-only cluster already cuts incremental build times; add + remote execution later for the full win on link-heavy full builds. + + + ## What's next - [Persistent workers](/deployment/persistent-workers) for the diff --git a/web/apps/docs/content/docs/deployment/kubernetes.mdx b/web/apps/docs/content/docs/deployment/kubernetes.mdx index 7933ab46d..f4b6efc40 100644 --- a/web/apps/docs/content/docs/deployment/kubernetes.mdx +++ b/web/apps/docs/content/docs/deployment/kubernetes.mdx @@ -41,7 +41,7 @@ spec: spec: containers: - name: nativelink - image: ghcr.io/tracemachina/nativelink:v1.4.0 + image: ghcr.io/tracemachina/nativelink:v1.6.1 args: ["/config/cas.json5"] ports: - containerPort: 50051 @@ -104,7 +104,7 @@ spec: spec: containers: - name: nativelink - image: ghcr.io/tracemachina/nativelink:v1.4.0 + image: ghcr.io/tracemachina/nativelink:v1.6.1 args: ["/config/worker.json5"] resources: requests: { cpu: 4, memory: 8Gi } @@ -172,6 +172,26 @@ directory in the source tree has: Clone the repo, point at your cluster, `helm install`. Tune from there. +## FAQ + + + + Yes — one binary can serve CAS, AC, scheduler, and worker from a + single config. Split into the three workloads above once you need to + scale or restart them independently. + + + The CAS owns persistent volumes and a stable identity per replica — + losing one loses cached blobs. Workers hold no state at all; kill and + reschedule them freely, which is exactly what the HPA does. + + + Yes. Every RE-API service is gRPC; an Ingress that downgrades to + plain HTTP/1.1 breaks the protocol. NGINX needs `grpc_pass`; any + modern Gateway API implementation handles it. + + + ## What's next - [Metrics](/deployment/metrics) — wire it into your Grafana. diff --git a/web/apps/docs/content/docs/deployment/metrics.mdx b/web/apps/docs/content/docs/deployment/metrics.mdx index 3b15536f4..57296b6f2 100644 --- a/web/apps/docs/content/docs/deployment/metrics.mdx +++ b/web/apps/docs/content/docs/deployment/metrics.mdx @@ -270,6 +270,30 @@ storage: out_of_order_time_window: 1h ``` +## FAQ + + + + No — NativeLink emits OTLP only; there is no scrape endpoint. Either + run an OpenTelemetry Collector and let Prometheus scrape the + Collector's exporter, or enable Prometheus's OTLP receiver + (`--web.enable-otlp-receiver`) and send straight to it. The + [quick start](#quick-start) wires up the first path. + + + Metrics are one of the few Business Source License modules. + Individual developer use is fine; shared, production, or commercial + settings need NativeLink Cloud, Enterprise, or an intentionally + inexpensive commercial license — see the + [license page](https://nativelink.com/license). + + + `nativelink_scheduler_queue_depth` — the signal the + [Kubernetes HPA example](/deployment/kubernetes) targets. CPU-based + scaling reacts too slowly because workers are mostly I/O-bound. + + + ## Additional resources - [Metrics deployment examples](https://github.com/TraceMachina/nativelink/tree/main/deployment-examples/metrics) diff --git a/web/apps/docs/content/docs/deployment/oci-object-storage.mdx b/web/apps/docs/content/docs/deployment/oci-object-storage.mdx index 381878bbe..380709535 100644 --- a/web/apps/docs/content/docs/deployment/oci-object-storage.mdx +++ b/web/apps/docs/content/docs/deployment/oci-object-storage.mdx @@ -137,3 +137,17 @@ would for the AWS backend. cache round-trip, a 2 GiB multipart upload (~400 parts) with full byte verification, multipart abort/cleanup, and 200-way concurrent load. It has not been benchmarked at sustained production scale or duration. + +## FAQ + + + + Yes — segment them with `key_prefix` (`cas/` and `ac/` in the example + above) rather than provisioning two buckets. + + + `aws`, `gcs`, `azure`, `ontap`, and `r2`, alongside `oci` — one store + type, six providers, sharing the same multipart, retry, and streaming + code paths. + + diff --git a/web/apps/docs/content/docs/deployment/on-prem-overview.mdx b/web/apps/docs/content/docs/deployment/on-prem-overview.mdx index 274c0e45b..d035b630d 100644 --- a/web/apps/docs/content/docs/deployment/on-prem-overview.mdx +++ b/web/apps/docs/content/docs/deployment/on-prem-overview.mdx @@ -107,7 +107,7 @@ contracts; truly global teams. Numbers we've seen in production. Yours will vary — these are the order-of-magnitude starting points: -| Resource | Per active developer / week | +| Resource | Per active developer | | --------------------- | ------------------------------- | | CAS storage (C++) | 15-20 GB | | CAS storage (Go/Rust) | 5-10 GB | @@ -118,6 +118,29 @@ order-of-magnitude starting points: When the cluster is healthy, the scheduler is the cheapest piece and CAS storage is the dominant cost. Plan accordingly. +## FAQ + + + + The CAS. Schedulers and workers are stateless, and the Action Cache + can be wiped harmlessly — losing it just forces re-execution until it + warms back up. That asymmetry drives backup policy: snapshot the CAS + backend, treat everything else as disposable. + + + On scheduler queue depth, not CPU — workers spend most of their time + blocked on I/O, so CPU-based scaling reacts too late. The + [Kubernetes guide](/deployment/kubernetes) has a working HPA + targeting `nativelink_scheduler_queue_depth`. + + + Filesystem CAS: `rsync` or ZFS/Btrfs snapshots. S3-compatible: + versioning, lifecycle policies, and cross-region replication for DR. + Redis: ephemeral by design — loss is a cache miss. The procedure is + in [NativeLink on-prem](/getting-started/on-prem). + + + ## What's next - [Kubernetes](/deployment/kubernetes) — working Helm chart. diff --git a/web/apps/docs/content/docs/deployment/persistent-workers.mdx b/web/apps/docs/content/docs/deployment/persistent-workers.mdx index 9ebdbddd0..61ce78d0e 100644 --- a/web/apps/docs/content/docs/deployment/persistent-workers.mdx +++ b/web/apps/docs/content/docs/deployment/persistent-workers.mdx @@ -110,6 +110,22 @@ clusters don't waste memory. a soft mitigation; for serious workloads, set a hard `max_invocations` and recycle. +## FAQ + + + + Rarely — plain `clang` starts in ~10 ms, so the win is negligible and + not worth the caveats. They pay back almost immediately on JVM-family + toolchains (`javac`, `kotlinc`) and modestly on `rustc`. + + + They can — a warm process isn't sandboxed by default, so a toolchain + that mutates global state between invocations produces + nondeterministic results. Sandbox the worker process itself and + recycle workers periodically (see [Caveats](#caveats)). + + + ## What's next - [Configuration → Production](/configuration/production) — the diff --git a/web/apps/docs/content/docs/explanations/architecture.mdx b/web/apps/docs/content/docs/explanations/architecture.mdx index eae6bbe12..e416e061e 100644 --- a/web/apps/docs/content/docs/explanations/architecture.mdx +++ b/web/apps/docs/content/docs/explanations/architecture.mdx @@ -140,6 +140,28 @@ Three operational properties drove the design: The same simplicity at the protocol level is why NativeLink can sustain over a billion build requests a month on modest hardware. +## FAQ + + + + Only the CAS — losing it loses every cached artifact. The AC is a + lookup table you can lose harmlessly: clients re-execute and refill + it. Schedulers and workers hold no durable state at all. + + + Yes — the same executable serves any subset, chosen by config. + Single-developer setups run everything in one process; teams split + the services so each scales independently. + + + A namespace within one cluster: action hashes from different instance + names never collide, so one deployment can serve isolated teams. + Every client and server must agree on it — a mismatch looks like a + permanently empty cache. More vocabulary in the + [glossary](/reference/glossary). + + + ## What's next - [Local Remote Execution](/explanations/lre) — running the same diff --git a/web/apps/docs/content/docs/explanations/history.mdx b/web/apps/docs/content/docs/explanations/history.mdx index 3631ac4d2..8659694fc 100644 --- a/web/apps/docs/content/docs/explanations/history.mdx +++ b/web/apps/docs/content/docs/explanations/history.mdx @@ -70,3 +70,22 @@ working examples, a clear path from `setup` to `production`. If you find a gap, the docs source lives in [`web/apps/docs/content/docs`](https://github.com/TraceMachina/nativelink) and PRs are welcome. + +## FAQ + + + + Three operational reasons: no GC pauses (tail latency comes from + network and storage, not the runtime), memory safety (a scheduler + memory bug could silently serve a wrong artifact org-wide — Rust + removes the class at compile time), and throughput (a single instance + sustains over a billion requests a month with sub-millisecond `p99` + lookups). + + + Compile times — mitigated by sccache backed by NativeLink itself — + and a smaller hiring pool, which has hurt less than expected: the + people who write distributed-systems Rust tend to be exactly the + people you want writing distributed systems. + + diff --git a/web/apps/docs/content/docs/explanations/lre.mdx b/web/apps/docs/content/docs/explanations/lre.mdx index ff7ad1d21..cac03a257 100644 --- a/web/apps/docs/content/docs/explanations/lre.mdx +++ b/web/apps/docs/content/docs/explanations/lre.mdx @@ -135,8 +135,35 @@ filesystem state. That's the entire reason. Nix gives us hash-pinned tools; everything else follows. -For background, see [What is Nix?](/faq/nix) and -[How do I make my Bazel setup hermetic?](/faq/hermeticity). +For background on Nix itself, see +[Develop with Nix](/contribute/nix); for the hermeticity checklist, +see the [FAQ](#faq) below. + +## FAQ + + + + Pin the toolchain (Nix or `rules_cc`'s toolchain abstraction — + never `/usr/bin/cc`), pin every `http_archive` URL and sha256, + add `build --incompatible_strict_action_env` to stop `$PATH` + leaks, enable strict include checking + (`--features=layering_check`), and compare action hashes across + machines with `bazel aquery`. LRE gives you the toolchain-pinning + piece for free. + + + Build the same target twice on the same machine under LRE and + compare outputs — anything that differs is a leak. At the cluster + level, non-hermetic actions surface as cache misses where you + expected hits, or as nondeterministic action results. + + + No — it solves determinism, not throughput. Fan-out across + hundreds of cores, or building for platforms you can't host, + still wants remote workers. The point is that action hashes + match, so local and remote execution share one cache. + + ## What's next diff --git a/web/apps/docs/content/docs/faq/architecture.mdx b/web/apps/docs/content/docs/faq/architecture.mdx deleted file mode 100644 index 53f7ae530..000000000 --- a/web/apps/docs/content/docs/faq/architecture.mdx +++ /dev/null @@ -1,70 +0,0 @@ ---- -title: How does NativeLink work under the hood? -description: Four services glued together by the Remote Execution API — CAS, Action Cache, Scheduler, and Workers — where only the CAS holds durable state. ---- - -**NativeLink is four pieces glued together by the Remote Execution -API.** The CAS stores every blob keyed by its SHA-256 hash; the -Action Cache (AC) maps hash(Action) → ActionResult; the Scheduler -dispatches `Execute()` calls to matching workers; Workers fetch -inputs, run the command, and upload outputs. Each service -scales independently, and one binary can run all four. The full -walkthrough with diagrams is -[Explanations → Architecture](/explanations/architecture). - -## CAS versus Action Cache - -The CAS is the only service you must persist — losing it loses every -cached artifact. The AC is a smaller lookup table saying "this exact -computation has been done before"; losing it is harmless, because -clients re-execute and refill it. That asymmetry drives most -operational decisions (backups, replica counts, storage tiers). - -## What happens on a cache hit vs. a miss - -The client hashes the action — source digests, compiler version, -command line — and asks the AC. On a hit, outputs come straight from -the CAS in milliseconds and execution is skipped entirely. On a -miss, the Scheduler dispatches to a worker whose -`platform_properties` satisfy the action's requirements; the worker -runs the command (on Linux, optionally isolated with kernel -namespaces via the `use_namespaces` and `use_mount_namespace` -worker options), uploads outputs, and the ActionResult lands in -the AC for next time. - -## Why "stateless" schedulers and workers matter - -Schedulers hold no durable state — run several behind a load -balancer and restart them at will. Workers are disposable — scaling -them out is the cheapest way to make builds faster, which is why -autoscaling targets workers and nothing else. - -## The vocabulary that makes the docs make sense - -- **Digest** — a content hash plus a size; used everywhere instead - of file paths, which is what makes content-addressing work. -- **Instance name** — a namespace within a cluster; hashes from - different instance names never collide, so one cluster can serve - isolated teams. -- **Platform properties** — key/value tags matched between actions - ("needs Linux, GPU") and workers ("has Linux, GPU") to decide - dispatch. - -The full list is in the [glossary](/reference/glossary). - -## Does this actually pay off in practice - -Yes — see the -[real-world example](/explanations/architecture#a-real-world-example) -of LLVM contributors cutting full clang builds from 17 minutes to 4. -Adoption history and production users are in -[History](/explanations/history). - -## Further reading - -- [What is remote execution?](/faq/remote-execution) — the concept, - from the client's point of view. -- [What is remote caching?](/faq/caching) — how the hash is - computed. -- [Why Rust?](/faq/rust) — the language choice behind the no-GC - latency profile. diff --git a/web/apps/docs/content/docs/faq/caching.mdx b/web/apps/docs/content/docs/faq/caching.mdx deleted file mode 100644 index c22a9772c..000000000 --- a/web/apps/docs/content/docs/faq/caching.mdx +++ /dev/null @@ -1,53 +0,0 @@ ---- -title: What is remote caching? -description: Reuse build outputs across machines so you only pay for compilation once. ---- - -Remote caching is the practice of storing the result of every build -action in a shared store, keyed by the hash of the action's inputs. If -anyone has already compiled the file you're about to compile — a -teammate, your CI, an agent — you fetch the result instead of doing -the work. - -## How the hash gets computed - -Take everything that affects a compilation's output: - -- The source files (each by content hash). -- The compiler binary (by content hash). -- The compile command, exactly as it'll be executed. -- The set of header files included transitively. -- Anything else the build system declared as an input. - -Hash that whole bundle. Two invocations with the same bundle produce -identical outputs; the cache returns the prior result. One byte -different anywhere — a header you didn't realise you depended on, a -patch version bump in the compiler — and the hash changes, the cache -misses, the work runs fresh. - -This only works if the build is **hermetic** — i.e. every input is -declared. See [hermeticity](/faq/hermeticity). - -## What a cache hit looks like - -A typical Bazel hit: - -``` -INFO: From Compiling src/foo.cc: -(remote cache hit) -Target //src:foo up-to-date: - bazel-bin/src/libfoo.a -``` - -The action ran in 2-10 ms. The compiler never started; the linker -never touched it. You paid the cost of one HTTP request. - -## Where NativeLink fits - -NativeLink implements the standard -[Remote Execution API](https://github.com/bazelbuild/remote-apis) — -the same protocol Bazel, Buck2, Siso, Goma, and Pants speak. You -point your build at a NativeLink server, and every action gets cached -automatically. - -The fast version: [Setup](/getting-started/setup). diff --git a/web/apps/docs/content/docs/faq/clients.mdx b/web/apps/docs/content/docs/faq/clients.mdx deleted file mode 100644 index 33d2c2060..000000000 --- a/web/apps/docs/content/docs/faq/clients.mdx +++ /dev/null @@ -1,65 +0,0 @@ ---- -title: Which build tools work with NativeLink? -description: Any client that speaks the Remote Execution API — Bazel, Buck2, Siso, Pants, BuildStream, Goma, and CMake via recc — with no build-file rewrites. ---- - -**Any build tool that implements the Remote Execution API can point -at NativeLink without rewriting build files.** That includes Bazel, -Buck2, Siso (Chromium's Ninja replacement), Pants, BuildStream, Goma, -and plain CMake projects via [`recc`](/getting-started/other-build-systems/cmake-recc). -The full guides live in -[Getting started → Setup](/getting-started/setup) and -[Other build systems](/getting-started/other-build-systems); this page -is the short version. - -## How each client is wired - -- **Bazel** — `.bazelrc` flags: - `build --remote_cache=grpc://localhost:50051`, - `--remote_executor=...`, and `--remote_instance_name=main`. -- **Buck2** — a `[buck2_re_client]` section in `.buckconfig` with - `engine_address`, `action_cache_address`, and `cas_address` all - pointing at the same listener, plus an execution platform with - `remote_enabled = True`. -- **Siso** — environment variables: `SISO_REAPI_ADDRESS`, - `SISO_REAPI_INSTANCE`, and `RBE_service_no_security=true` for - local, non-TLS development. -- **Pants** — `remote_cache_read`, `remote_cache_write`, - `remote_store_address`, and `remote_instance_name` under - `[GLOBAL]` in `pants.toml`. -- **BuildStream** — `artifacts.servers` (with `push: true`) and the - `remote-execution` service URLs. -- **CMake** — set `recc` as the compiler launcher - (`CMAKE_C_COMPILER_LAUNCHER`) in cache-only mode; compiles happen - locally and outputs travel through NativeLink. - -All clients targeting the same cache must agree on the instance name -— see [Why isn't my build hitting the cache?](/faq/troubleshooting) -for what happens when they don't. - -## What platforms can I run the server on - -- **Linux (x86_64 and ARM64)** — Docker image - (`ghcr.io/tracemachina/nativelink`, multi-arch) or Nix. -- **macOS (Apple Silicon and Intel)** — the Docker image runs - natively on both architectures; the Nix path builds from source - (needs 15-20 GB of free disk) for a native binary. -- **Windows (x86_64 and ARM64)** — via Docker, or WSL2 following the - Linux instructions. - -## How do I confirm it's actually working - -Rebuild without changing source. With Bazel, -`--execution_log_json_file=/tmp/exec.json` lets you count hits with -`grep -c '"remoteCacheHit": true'`; unchanged targets report -`(remote cache hit)` in the console. `recc` prints an -`Action Cache hit` line at `RECC_LOG_LEVEL=info`. - -## Further reading - -- [Setup](/getting-started/setup) — running a server in under ten - minutes. -- [Classic RBE examples](/rbe/examples) — cache-only vs. full remote - execution vs. hybrid, and which to start with. -- [What is remote caching?](/faq/caching) — how the cache hash is - computed. diff --git a/web/apps/docs/content/docs/faq/configuration.mdx b/web/apps/docs/content/docs/faq/configuration.mdx deleted file mode 100644 index a43ff40f6..000000000 --- a/web/apps/docs/content/docs/faq/configuration.mdx +++ /dev/null @@ -1,68 +0,0 @@ ---- -title: How do I configure NativeLink? -description: JSON5 files with four top-level sections — stores, servers, schedulers, and workers. Start cache-only and add execution later. ---- - -**NativeLink is configured with JSON5 files.** A single file per -cluster is the simplest way to start, but production deployments -usually split into one config per service so the CAS, scheduler, and -workers can scale independently — the deployment examples show both -shapes. JSON5 specifically, because it supports comments, trailing -commas, and single quotes — so operational notes live next to the -config instead of drifting in a separate wiki. The conceptual walkthrough is -[Configuration → Introduction](/configuration/intro); every knob and -default is in the autogenerated -[configuration reference](/reference/nativelink-config). - -## The four top-level sections - -- **`stores`** — named storage backends for CAS and Action Cache - data. See [Which store backend should I use?](/faq/stores) -- **`servers`** — gRPC/HTTP listeners exposing RE-API services. A - cache-only node enables `cas`, `ac`, `bytestream`, and - `capabilities`; an executor node adds `execution` and - `worker_api`. -- **`schedulers`** — optional; required only if the node accepts - `Execute()` calls. Matches actions to workers via - `supported_platform_properties`. -- **`workers`** — optional; required only if the node runs actions. - Each worker names its scheduler (`worker_api_endpoint`), its store - (`cas_fast_slow_store`), and the `platform_properties` describing - what it can run. - -## What's the simplest working config - -An in-memory CAS + AC pair behind one listener — the -`basic_cas.json5` example in the repo's -`nativelink-config/examples/`. Everything lives in RAM and vanishes -on restart: fine for a demo or a short-lived CI runner, disastrous -as a team cache. To survive restarts, switch the stores from -`memory` to `filesystem` with a `content_path`, `temp_path`, and an -`eviction_policy.max_bytes` -([Basic configurations](/configuration/basic)). - -## How do I add remote execution to a cache-only config - -Three additions: a `schedulers` block (a `simple` scheduler with -`supported_platform_properties`), `execution` + `worker_api` entries -in the server's `services`, and a `workers` array entry pointing at -the scheduler. The full diff is shown in -[Basic configurations](/configuration/basic). - -## What does a production config look like - -A sharded two-tier CAS (Redis in front of S3 via `fast_slow`, split -with a `shard` store), a small durable Action Cache, 2-3 stateless -schedulers behind a load balancer, mTLS between components, and -OTLP metrics enabled. A complete working example ships in the repo -as `nativelink-config/examples/production.json5` — start there and -trim ([Production configurations](/configuration/production)). - -## Further reading - -- [Content-defined chunking](/configuration/chunking) — cut transfer - bytes 80-90% for incrementally-changing artifacts. -- [Remote cache compression](/configuration/compression) — zstd on - the wire, enabled with one capabilities flag. -- [What are toolchains?](/faq/toolchains) — the concepts behind - `platform_properties`. diff --git a/web/apps/docs/content/docs/faq/contributing.mdx b/web/apps/docs/content/docs/faq/contributing.mdx deleted file mode 100644 index 7199637b9..000000000 --- a/web/apps/docs/content/docs/faq/contributing.mdx +++ /dev/null @@ -1,59 +0,0 @@ ---- -title: How do I contribute to NativeLink? -description: Open an issue first, develop with Bazel, Cargo, or Nix, sign every commit with DCO, and expect a first review response within two business days. ---- - -**Open an issue before anything bigger than a typo fix, then pick -whichever dev loop fits your change.** The complete rules live in -[Contribution guidelines](/contribute/guidelines); this is the -short version. - -## Which dev workflow should I use - -- **Cargo** ([guide](/contribute/cargo)) — fastest into a small, - localized change. `cargo build --workspace`, `cargo test -p - `. The Rust toolchain is pinned in `rust-toolchain.toml`; - `protoc` 3+ must be on PATH. -- **Bazel** ([guide](/contribute/bazel)) — what CI runs. - `bazel test //...` (first build 10-20 minutes, seconds after). - Editor integration via `bazel run //tools:gen-rust-project`. -- **Nix** ([guide](/contribute/nix)) — `nix develop` reproduces the - exact CI toolchain (Cargo, Bazel, protoc, mold, clang, linters). - Required for LRE work; needs 15-20 GB free disk on first run. - -A green `cargo test --workspace` does not guarantee green CI — some -crates are Bazel-only, and CI runs Bazel. Run `bazel test //...` -before submitting if you touched anything Bazel-specific. - -## What the review process expects - -- **DCO sign-off on every commit** (`git commit -s`) — this is the - formal license declaration for your contribution. -- **Formatter and linter clean**: `cargo fmt --all`, - `cargo clippy --all-targets`. -- **One change per PR**, imperative-mood title, a body that explains - *why*, and tests for new behavior. -- First response within 2 business days (ping the PR if slower); one - reviewer per change (others sometimes chime in); every merge is - squashed to one commit on `main`. - -Accepted: bug fixes with regression tests, performance work with -reproducible benchmarks, new storage backends/schedulers/worker -types (issue first), docs, and tooling. Rejected: RE-API breaking -changes, single-vendor code paths, and refactoring for its own sake. - -## How do I contribute to the docs or website - -The docs live in `web/apps/docs/content/docs/` as MDX; -`cd web && bun install && bun dev:docs` serves them at -localhost:3001/docs with live reload. Frontmatter needs `title` and -`description`; sidebar order comes from each section's `meta.json`. -Style rules and available components are in -[Working on documentation](/contribute/docs). - -## Further reading - -- [Contribution guidelines](/contribute/guidelines) — the full - accepted/rejected list and licensing details. -- [Is NativeLink free?](/faq/cost) — how the FSL + BSL module - licensing fits together, including contributor waivers. diff --git a/web/apps/docs/content/docs/faq/cost.mdx b/web/apps/docs/content/docs/faq/cost.mdx deleted file mode 100644 index b4a594b97..000000000 --- a/web/apps/docs/content/docs/faq/cost.mdx +++ /dev/null @@ -1,60 +0,0 @@ ---- -title: Is NativeLink free? -description: Yes — NativeLink is source-available and free for permitted self-hosted use. Cloud and Enterprise are paid offerings. ---- - -**Yes — NativeLink is source-available and free for permitted -self-hosted use.** Most of the monorepo is licensed under -`FSL-1.1-Apache-2.0`, which allows internal use, modification, and -redistribution for non-competing purposes, then grants an Apache 2.0 -future license on the schedule described in the repository -[`LICENSE`](https://github.com/TraceMachina/nativelink/blob/main/LICENSE). -The customer-facing summary lives on the -[NativeLink license page](https://nativelink.com/license). - -NativeLink is not a single-license monorepo. Some feature modules -carry their own source headers. In particular, metrics and remote -persistent workers are licensed under the Business Source License. -Developers using NativeLink for an individual cache do not need a -commercial license. Teams using those modules in shared, production, -or commercial settings can use NativeLink Cloud, Enterprise, or a -separate commercial license, which is intentionally very inexpensive. -Meaningful contributors may be eligible for license waivers; contact -the maintainers before relying on one. - -If you'd rather pay someone to run it for you, two paid tiers exist: - -- **Cloud** — managed multi-tenant cluster, flat $999+/month. Full - SLA, autoscaling, dashboard. -- **Enterprise** — single-tenant deployment, dedicated solutions - engineer, on-prem or managed. Custom contracts. - -The -[pricing page](https://nativelink.com/pricing) has the feature -comparison. - -## What does "free" actually cover - -- The full RE-API server: CAS, AC, scheduler, worker. -- Every storage backend: filesystem, S3, Redis, GCS, Azure Blob. -- Every supported build client: Bazel, Buck2, Siso, Pants, Goma. -- The CLI tooling and Helm chart. - -No artificial limits on cache size, action count, worker count, or -team size. The self-hosted build is the same codebase we run in -production, subject to the license that applies to each module, minus -the operational tooling and dedicated support that Cloud and -Enterprise add. - -## What you'd pay for instead - -Hosting NativeLink yourself costs: - -- Compute for the control plane (typically a few small VMs). -- Storage for the CAS (S3 / equivalent, sized for ~5-20 GB per active - developer per week). -- Worker compute (variable; this is the part Cloud rents to you). -- Engineer hours running it. - -For most teams under 30 engineers, self-host is cheaper. Past that, -Cloud usually wins on engineer-time alone. diff --git a/web/apps/docs/content/docs/faq/deployment.mdx b/web/apps/docs/content/docs/faq/deployment.mdx deleted file mode 100644 index 7a4a877d4..000000000 --- a/web/apps/docs/content/docs/faq/deployment.mdx +++ /dev/null @@ -1,68 +0,0 @@ ---- -title: How do I run NativeLink in production? -description: Pick one of three reference architectures, deploy on Kubernetes or plain VMs, autoscale workers on queue depth, and size storage per developer. ---- - -**Most teams land on one of three shapes.** A single-node cache (one -machine, one binary, no remote execution — fine up to ~10 -developers), a cache cluster with autoscaling workers (Redis hot -tier + S3 durable tier behind a load balancer — the standard -production deployment for 50-500 engineers), or a multi-region -cluster with per-region storage for regulated environments. The -decision guide is [On-prem overview](/deployment/on-prem-overview); -whether to self-host at all is covered in -[NativeLink on-prem](/getting-started/on-prem). - -## What do I actually deploy - -Four services: CAS (stateful — the only thing you must persist), Action -Cache (stateful but disposable), Scheduler (stateless — run 2-3 -behind a load balancer), and Workers (stateless and disposable — -autoscaling them is the cheapest speedup). One binary serves all -four; split them into separate processes beyond single-developer -scale. See [How does NativeLink work?](/faq/architecture) - -On Kubernetes that maps to a CAS StatefulSet (3+ replicas, one PVC -each, composed into one addressable store via the `shard` backend), -a scheduler Deployment, and an autoscaled worker Deployment. A -reference Helm chart with development, staging, and production -values lives in the repo's `deployment-examples/kubernetes/` -([Kubernetes](/deployment/kubernetes)). The Ingress in front must -speak gRPC end-to-end. - -## How should workers scale - -On queue depth, not CPU. Workers spend most of their time blocked on -I/O, so CPU-based scaling reacts too slowly. The reference HPA -targets the `nativelink_scheduler_queue_depth` metric (example: -`minReplicas: 4`, `maxReplicas: 200`, target average value 20). - -## How much capacity do I need - -Per active developer: 15-20 GB CAS storage for C++, 5-10 GB -for Go/Rust. Worker CPU: ~1 vCPU per concurrent action. Scheduler -memory: ~100 MB per 1,000 in-flight actions. At least 1 Gbps between -workers and CAS — cache reads are the hot path. CAS storage -dominates cost in a healthy cluster. - -## What about backups - -Depends on the CAS backend: - -- **Filesystem** — `rsync` or ZFS/Btrfs snapshots. -- **S3-compatible** — versioning, lifecycle policies, and - cross-region replication for DR. -- **Redis** — treat as ephemeral; loss is a cache miss. - -The Action Cache can be wiped without data loss. - -## Further reading - -- [Production configurations](/configuration/production) — the - config side: sharded stores, mTLS, HA schedulers. -- [How do I monitor NativeLink?](/faq/observability) — metrics and - alert thresholds. -- [Persistent workers](/deployment/persistent-workers) — 5-20x for - JVM-style toolchains; not worth it for plain clang. -- [Chromium](/deployment/chromium) — a worked example with expected - cache-hit rates. diff --git a/web/apps/docs/content/docs/faq/hermeticity.mdx b/web/apps/docs/content/docs/faq/hermeticity.mdx deleted file mode 100644 index e9ae2844a..000000000 --- a/web/apps/docs/content/docs/faq/hermeticity.mdx +++ /dev/null @@ -1,68 +0,0 @@ ---- -title: How do I make my Bazel setup hermetic? -description: Eliminate every "it works on my machine" by declaring every input. ---- - -A hermetic build is one where every input is declared, every output -is reproducible, and the result depends only on the declared inputs. -Same recipe, same ingredients, same dish, every time. - -Bazel **can** produce hermetic builds. Out of the box, it doesn't — -most Bazel projects depend implicitly on system tooling, environment -variables, and the host OS. Reaching real hermeticity takes -deliberate work. - -## The hermeticity checklist - - -
  • - **Pin the toolchain.** Don't rely on `/usr/bin/cc`. Use - `rules_cc`'s toolchain abstraction or a Nix-provided toolchain; - register the platform in `MODULE.bazel`. -
    • -
  • - **Lock dependency sources.** Every `http_archive` should pin both - the URL and the sha256. Module dependencies should pin a version, - not a range. -
    • -
  • - **Forbid `$PATH` leaks.** Add - `build --incompatible_strict_action_env` to your `.bazelrc`. - Bazel will now only forward an explicitly-allowed set of env - vars. -
    • -
  • - **Use the sandbox you trust.** On Linux: `linux-sandbox`. On - macOS: `darwin-sandbox`. For maximum isolation, run actions in a - container or under [LRE](/explanations/lre). -
    • -
  • - **Declare every header.** Strict include-checking - (`--features=layering_check`) catches the headers your `cc_library` - silently transitively depends on. -
    • -
  • - **Compare action hashes across machines.** If the same target - produces different hashes on two laptops, something is leaking. - `bazel aquery 'mnemonic("CppCompile", //target:label)' --output=text` - shows you the action's full input set. -
    • -     - -## How NativeLink helps - -A hermetic build maps directly to a content-addressed cache: same -inputs = same hash = same cached result. NativeLink doesn't *make* -your build hermetic, but it surfaces failures fast — non-hermetic -actions show up as cache misses where there should be hits, or, worse, -as nondeterministic action results. - -The fastest way to find leaks: enable -[LRE](/explanations/lre), build the same target twice on the -same machine, and compare outputs. Anything that differs is a leak. - -## Further reading - -- [What are toolchains?](/faq/toolchains) -- [What is Nix?](/faq/nix) -- [LRE](/explanations/lre) — the easiest way to get there. diff --git a/web/apps/docs/content/docs/faq/lre.mdx b/web/apps/docs/content/docs/faq/lre.mdx deleted file mode 100644 index d781c04d5..000000000 --- a/web/apps/docs/content/docs/faq/lre.mdx +++ /dev/null @@ -1,48 +0,0 @@ ---- -title: What is LRE? -description: Local Remote Execution — the hermeticity of RBE without the network. ---- - -LRE (Local Remote Execution) runs a NativeLink worker on your laptop -and routes every build action through it. You get the hermeticity -guarantees of remote execution — content-addressed inputs, pinned -toolchain, sandboxed processes — without sending anything over the -network. - -## What it solves - -The classic developer-experience trade-off: - -- **Pure local build** — fast feedback, "works on my machine" bugs. -- **Pure remote build** — bit-identical with CI, slow on every change. - -LRE is the third option: bit-identical with CI, fast on every change. -Cache hits from the remote cluster apply locally; cache writes from -local apply remotely. - -## How it works - -Three pieces: - -1. A NativeLink server bound to `localhost`. -2. A Nix-pinned toolchain: every binary the build invokes has a - content-addressed path in the Nix store. -3. A configured build client (Bazel, Buck2, ...) targeting the local - server. - -The build issues the same `Execute` RPCs it would issue against a -remote cluster — they just happen to land on `localhost`. - -## When to use LRE - -| Scenario | LRE | -| ---------------------------------------------- | ---- | -| Solo dev iterating, want CI reproducibility | ✓ | -| Team sharing a cache without a server | ✓ | -| Repro a CI failure locally | ✓ | -| Fan-out across 200 cores | ✗ | -| Building for a platform you can't host | ✗ | - -The split: LRE for determinism, remote workers for throughput. - -The full write-up: [Explanations → LRE](/explanations/lre). diff --git a/web/apps/docs/content/docs/faq/meta.json b/web/apps/docs/content/docs/faq/meta.json deleted file mode 100644 index e9e56b43c..000000000 --- a/web/apps/docs/content/docs/faq/meta.json +++ /dev/null @@ -1,21 +0,0 @@ -{ - "pages": [ - "cost", - "caching", - "remote-execution", - "architecture", - "lre", - "toolchains", - "hermeticity", - "nix", - "rust", - "clients", - "configuration", - "stores", - "deployment", - "observability", - "troubleshooting", - "contributing" - ], - "title": "FAQ" -} diff --git a/web/apps/docs/content/docs/faq/nix.mdx b/web/apps/docs/content/docs/faq/nix.mdx deleted file mode 100644 index 07c0b07ee..000000000 --- a/web/apps/docs/content/docs/faq/nix.mdx +++ /dev/null @@ -1,49 +0,0 @@ ---- -title: What is Nix? -description: A purely functional package manager that gives NativeLink its hermeticity guarantees. ---- - -Nix is a package manager that builds every package in isolation, with -a hash that encodes every input — source code, dependencies, build -flags, even the version of `gcc` used to compile the dependencies. -Two machines running the same Nix flake produce bit-identical -artifacts. - -That property — hash-pinned, content-addressed everything — is what -NativeLink needs for [LRE](/explanations/lre) and is why we use -Nix to provision toolchains. - -## What it gives you, concretely - -- **Toolchains in the Nix store.** Your `clang` lives at - `/nix/store/abc123-clang-18.1.6/bin/clang`. The path itself is the - hash. NativeLink can use that path as an input identifier directly. -- **No "global" state to leak.** A Nix shell exposes only the - packages you ask for. There's no `/usr/lib` to accidentally - depend on. -- **Reproducible across machines.** `nix develop` on your laptop and - on CI produces the same shell, byte-for-byte. - -## What you give up - -- A learning curve. The Nix language is unfamiliar; the error - messages take getting used to. -- Initial download time. The first `nix develop` for a big project - fetches gigabytes. - -## How to start - -The -[next-gen Nix installer](https://github.com/NixOS/experimental-nix-installer) -is the smoothest path. After installing: - -```bash -git clone https://github.com/TraceMachina/nativelink -cd nativelink -nix develop -``` - -You're now in a shell with the exact toolchain CI uses — Rust, Bazel, -mold, `protoc`, the works. - -For a full LRE setup, see [Explanations → LRE](/explanations/lre). diff --git a/web/apps/docs/content/docs/faq/observability.mdx b/web/apps/docs/content/docs/faq/observability.mdx deleted file mode 100644 index e9e74bb90..000000000 --- a/web/apps/docs/content/docs/faq/observability.mdx +++ /dev/null @@ -1,58 +0,0 @@ ---- -title: How do I monitor NativeLink? -description: NativeLink emits OTLP metrics only — route them through an OpenTelemetry Collector or Prometheus's OTLP receiver, and opt in to cache metrics with the cache_metrics store wrapper. ---- - -**NativeLink emits OTLP metrics only — there is no direct Prometheus -scrape endpoint.** Two working paths: send OTLP to an OpenTelemetry -Collector and let Prometheus scrape the Collector's exporter, or -send OTLP/HTTP straight to Prometheus with its OTLP receiver enabled -(`--web.enable-otlp-receiver`). Full wiring, dashboards, and a -Docker Compose quickstart are in -[Metrics & observability](/deployment/metrics). - -## Why am I seeing no cache metrics - -Cache metrics are opt-in: you must wrap the CAS/AC store with the -`cache_metrics` store wrapper in config. Enabling OTEL alone only -gets you execution-pipeline metrics. Without the wrapper NativeLink -builds the same store graph with zero metrics overhead. - -Note the licensing carve-out: metrics are one of the few modules -under the Business Source License — individual developer use is -fine, shared/production use needs Cloud, Enterprise, or an -(intentionally inexpensive) commercial license. See -[Is NativeLink free?](/faq/cost) - -## What should I alert on - -Starting thresholds from the reference alert rules: - -- **Error rate** above 5% for 5 minutes — check worker failures and - scheduler logs. -- **Queue backlog** above 100 queued actions for 15 minutes — add - workers, or check worker matching. -- **Cache eviction rate** high for 10 minutes — increase storage or - tune the eviction policy. - -From the production-config side, also watch -`nativelink_cas_request_duration_seconds_p99` (sub-millisecond when -healthy; >50 ms is page-worthy) and -`nativelink_worker_connected_count` (sudden drops point at the -scheduler). - -## Metrics aren't showing up at all - -Three checks, in order: `env | grep OTEL_` for the exporter -variables, the Collector's health endpoint (`:13133/health`), and -the Collector's own metrics (`:8888/metrics`, grep -`otelcol_receiver`) to confirm data is arriving. Prometheus -complaining about out-of-order samples wants a larger -`storage.tsdb.out-of-order-time-window` (the quickstart uses 30m). - -## Further reading - -- [Metrics & observability](/deployment/metrics) — every metric and - label, plus Grafana dashboards. -- [How do I run NativeLink in production?](/faq/deployment) — the - queue-depth metric doubles as the worker autoscaling signal. diff --git a/web/apps/docs/content/docs/faq/remote-execution.mdx b/web/apps/docs/content/docs/faq/remote-execution.mdx deleted file mode 100644 index 2cf130d5b..000000000 --- a/web/apps/docs/content/docs/faq/remote-execution.mdx +++ /dev/null @@ -1,62 +0,0 @@ ---- -title: What is remote execution? -description: Build steps run on a worker fleet, not your laptop. Faster builds, lower bills. ---- - -Remote execution ships the build action — inputs, command, sandbox -spec — to a fleet of workers and brings back the outputs. Your laptop -or CI runner becomes a thin coordinator; the heavy lifting happens on -hardware sized for it. - -## The shape - -``` -[ client ] ─ Execute() ─▶ [ scheduler ] - │ - ┌───────┼───────┐ - ▼ ▼ ▼ - [worker] [worker] [worker] - │ │ │ - └───────┴───────┘ - ▼ - [ CAS / AC ] -``` - -The client tells the scheduler "run this action." The scheduler picks -a worker that satisfies the action's platform requirements. The -worker fetches inputs from the content-addressed store, runs the -command in a sandbox, uploads outputs back to the store, and reports -the result. - -The cache from [remote caching](/faq/caching) sits underneath -all of this — actions hit it first; only misses get executed. - -## Why ship the work elsewhere - -Three reasons, in roughly the order teams adopt them: - -1. **Throughput.** A worker fleet has more cores than any one - developer machine. Linker steps that take 60 seconds on a laptop - take 8 on a hot worker. -2. **Hermeticity.** The worker has exactly one job, in exactly one - environment. No conflicting versions of system libraries, no - stale `/tmp`, no surprises. -3. **Cost.** CI runners are expensive because they sit idle most of - the day. A worker fleet utilises hardware constantly — average - utilisation goes from 5-10% to 60-80%. - -## Where NativeLink fits - -NativeLink is a Remote Execution API server. Any build tool that -speaks the protocol — -[Bazel](https://bazel.build), -[Buck2](https://buck2.build), -[Siso](https://chromium.googlesource.com/build/+/refs/heads/main/siso/README.md), -[Pants](https://www.pantsbuild.org), -[Goma](https://chromium.googlesource.com/infra/goma/client/) — -plugs in with no code changes. - -The flow looks like the diagram above, with NativeLink as the -scheduler + worker fleet. See -[Architecture](/explanations/architecture) for the full -walkthrough. diff --git a/web/apps/docs/content/docs/faq/rust.mdx b/web/apps/docs/content/docs/faq/rust.mdx deleted file mode 100644 index 9e2a7b6c3..000000000 --- a/web/apps/docs/content/docs/faq/rust.mdx +++ /dev/null @@ -1,44 +0,0 @@ ---- -title: Why Rust? -description: Memory safety without garbage collection — the right shape for a service that has to be fast and never lie. ---- - -## Three reasons - -### No GC pauses - -Build infrastructure runs under sustained load. A scheduler hiccup -during a `p99` request is the difference between "the build felt -fast" and "the build was unpredictable." Garbage-collected runtimes -insert that hiccup at exactly the wrong time, sized roughly to the -heap. - -Rust has no garbage collector. The tail latency on a NativeLink -cluster comes from network and storage, not the language runtime. - -### Memory safety - -A miscompiled binary that ships across an organisation is a serious -problem. Worse, build systems are pure-function critical -infrastructure: a memory bug in the scheduler could silently produce -the wrong artifact and serve it from cache to every developer. - -Rust catches those bugs at compile time. The class of incident -involving "we shipped something that wasn't what we compiled" doesn't -happen. - -### Throughput - -A single NativeLink instance handles over a billion build requests -per month on modest hardware. That's `~400 req/s` sustained, -`p99 < 1ms` lookups. Hitting those numbers in a GC'd language -requires more horizontal scaling, more memory, more tuning. - -## What it costs - -- Compile times. Cargo builds are slower than `go build`. Mitigated - by sccache + NativeLink itself. -- Smaller talent pool. Hiring Rust engineers is harder than hiring Go - engineers. We've found this less painful than expected — the people - who write distributed-system Rust tend to be exactly the people you - want writing distributed systems. diff --git a/web/apps/docs/content/docs/faq/stores.mdx b/web/apps/docs/content/docs/faq/stores.mdx deleted file mode 100644 index 7800662ef..000000000 --- a/web/apps/docs/content/docs/faq/stores.mdx +++ /dev/null @@ -1,70 +0,0 @@ ---- -title: Which store backend should I use? -description: Memory for demos, filesystem for single nodes, a cloud object store for anything multi-node — then compose wrappers like verify, compression, and fast_slow on top. ---- - -**Pick the terminal store by durability and sharing needs, then -compose wrapper stores on top.** A store is a named backend declared -once in the top-level `stores` array; its type is whichever key -appears inside the object. The prose companion to the generated -reference is [Store overview](/reference/nativelink-config/store-overview). - -## Terminal stores (hold actual bytes) - -- **`memory`** — fastest, wiped on restart. Dev setups and fast - tiers. -- **`filesystem`** — survives restarts, single-node. Small-team - caches and CI runners with persistent volumes. -- **`experimental_cloud_object_store`** — durable and shared across - nodes; one store type with six providers (`aws`, `gcs`, `azure`, - `ontap`, `r2`, `oci`). -- **`redis_store`** — shared, low-latency hot tier; pair it with - `size_partitioning` since Redis caps uploads around 256-512 MB. -- **`experimental_mongo`** — durable, shared, supports scheduler - change streams. - -## Wrapper stores (add behavior in front of another store) - -Wrappers nest arbitrarily deep. The shape most self-hosted -production clusters converge on, outermost first: - -- **`verify`** — hash + size checking, wrapping -- **`compression`** — lz4, wrapping -- **`fast_slow`** — where `fast` is a small in-memory tier and - `slow` is the durable cloud object store. - -Sharing one backend between two store trees? Use `ref_store` to -reference a store declared elsewhere by name instead of declaring it -twice. - -## The gotchas worth knowing up front - -- **`fast_slow` doesn't guarantee durability.** It never checks that - an object in `fast` also exists in `slow`. For artifacts that must - survive a fast-tier wipe (remote execution outputs), write through - both tiers deliberately. -- **`dedup` goes inside `compression`, not the other way around.** - Compressing before deduplication makes content-identical chunks - compress slightly differently, and deduplication stops working. -- **Some wrappers are CAS-only or AC-only.** `existence_cache` and - `size_partitioning` are CAS-only; `completeness_checking` is - AC-only. Misplacing them isn't a config error — it's a confusing - correctness bug. - -## How big should each piece be - -Rules of thumb from [On-prem overview](/deployment/on-prem-overview): -5-20 GB of CAS storage per active developer (C++ high, Go -low), Redis sized for the last 24-48 hours of artifacts (~20-40 GiB -for 1 TB/day of traffic), and an Action Cache typically under 1% of -CAS by bytes — losing it is harmless but forces rebuilds until it -warms back up. - -## Further reading - -- [Production configurations](/configuration/production) — the full - sharded S3 + Redis composition. -- [Basic configurations](/configuration/basic) — filesystem and - compression starters. -- [Store overview](/reference/nativelink-config/store-overview) — - every terminal and wrapper store, with JSON5 snippets. diff --git a/web/apps/docs/content/docs/faq/toolchains.mdx b/web/apps/docs/content/docs/faq/toolchains.mdx deleted file mode 100644 index 248c38aa9..000000000 --- a/web/apps/docs/content/docs/faq/toolchains.mdx +++ /dev/null @@ -1,52 +0,0 @@ ---- -title: What are toolchains? -description: How NativeLink represents compilers, linkers, and the runtime they need. ---- - -A toolchain is the bundle of binaries and configuration NativeLink -needs to run a build action — the compiler, the linker, the standard -library headers, the `cc` wrapper script, any shared library the -compiler depends on. Everything that touches the action. - -For caching and hermeticity to work, the toolchain has to be **fully -declared as inputs**. If the build implicitly picks up `clang` from -`$PATH`, two machines with slightly different `clang` versions produce -different outputs but identical action hashes. The cache lies; you -debug for an afternoon. - -## Three ways to provide one - -| Provider | Hermetic? | Setup cost | -| ------------------------- | ------------ | -------------- | -| **Nix flake** | Fully | Highest | -| **Bazel rules_cc / rules_rust** | Mostly | Medium | -| **System (`$PATH`)** | No | Lowest | - -### Nix - -The recommended path for [LRE](/explanations/lre) and any -deployment where reproducibility matters. The toolchain is identified -by a hash of its inputs; two machines pulling the same flake get -byte-identical binaries. - -### Bazel rules_cc / rules_rust - -The Bazel-native option. Pin the toolchain version in your -`MODULE.bazel`, register it as a platform, point actions at it. -Reasonable for most Bazel monorepos. - -### System tooling - -Use whatever's installed. Quick to set up, no hermeticity guarantees. -Reasonable for "I just want to try the cache out for a day." - -## Tagging actions with a toolchain - -NativeLink matches actions to workers via `platform_properties`. A -worker that has the `nvidia-a100` GPU tagged will be the only worker -that receives actions requesting that GPU. Same applies to toolchain -versions, OS, CPU count, custom labels — anything the cluster -operator configures. - -For the configuration knobs see -[Configuration → Production](/configuration/production#scheduler-ha-with-sticky-workers). diff --git a/web/apps/docs/content/docs/faq/troubleshooting.mdx b/web/apps/docs/content/docs/faq/troubleshooting.mdx deleted file mode 100644 index b66f48ede..000000000 --- a/web/apps/docs/content/docs/faq/troubleshooting.mdx +++ /dev/null @@ -1,67 +0,0 @@ ---- -title: Why isn't my build hitting the cache? -description: Almost always an instance-name or toolchain mismatch between the client that wrote and the client that reads — plus a handful of known silent failure modes. ---- - -**Cache writes succeed but reads always miss? Compare instance names -and toolchains first.** Action hashes computed under different -`instance_name` values, or by differently-configured toolchains, -never collide — so the cache looks empty even though it's full. This -isn't an error anywhere; it's silent. - -## The usual suspects, in order - -- **Instance-name mismatch.** Every client targeting a cache must - use the same instance name (`--remote_instance_name` for Bazel). - A server config that omits `instance_name` defaults to `""` — the - same as Bazel's default — but if the server sets one, a client - that doesn't send it fails with - `'instance_name' not configured for ''`. -- **Toolchain drift.** A cache entry written by one compiler version - is invisible to another. Chromium builds hit this as - "writes succeed, reads always miss." See - [What are toolchains?](/faq/toolchains) -- **Only half the flags set.** Bazel needs `--remote_cache` even - when `--remote_executor` is set (they can be the same address) — - otherwise `Execute` is called with a digest that was never - uploaded and fails with `Action ... is missing from CAS`. Pants - similarly needs both `remote_cache_read` and `remote_cache_write`. - -## My actions queue forever and nothing happens - -The scheduler's `supported_platform_properties` must be satisfiable -by at least one worker's `platform_properties`, or matching actions -sit queued indefinitely with no error surfaced. Actions falling back -to *local* execution are the same disease in different clients: -Buck2 with `remote_enabled = False` on the selected platform, Siso -requesting properties no worker advertises. - -## Known sharp edges - -- **Docker on SELinux-enforcing distributions** (Fedora, RHEL): the mounted config - needs an explicit `:Z` label or you get "Permission denied on - /config". -- **LRE: `nix develop` did nothing.** The flake module checks for a - `.git` directory and silently skips installation without one — - `git init && git add -A` first - ([Local Remote Execution](/rbe/local-remote-execution)). -- **LRE: `Unable to resolve host TODO`.** The generated - `user.bazelrc` ships with literal `TODO` placeholders for the - cache/executor endpoints; the error actually confirms everything - else worked — fill them in. -- **Bazel 9.1.0 + chunking + `--disk_cache`** corrupts outputs; use - 9.1.1+ ([Content-defined chunking](/configuration/chunking)). - -## How do I see what's actually happening - -Bazel: `--execution_log_json_file=/tmp/exec.json`, then grep for -`remoteCacheHit`. Siso: the siso log shows the platform properties -each action requested. Cluster-side: -[How do I monitor NativeLink?](/faq/observability) - -## Further reading - -- [Setup → Troubleshooting](/getting-started/setup) — per-client - wiring checks. -- [Local cache and executor](/rbe/local-testing) — a known-good - single-machine setup to test against. diff --git a/web/apps/docs/content/docs/getting-started/on-prem.mdx b/web/apps/docs/content/docs/getting-started/on-prem.mdx index fb47d20ae..c7c998218 100644 --- a/web/apps/docs/content/docs/getting-started/on-prem.mdx +++ b/web/apps/docs/content/docs/getting-started/on-prem.mdx @@ -66,7 +66,7 @@ each independently.
  • **Plan capacity.** Heuristics from production clusters: - - CAS storage: 5–20 GB per active developer per week, depending on + - CAS storage: 5–20 GB per active developer, depending on language. C++ skews high; Go skews low. - Worker CPU: 1 vCPU per concurrent action. Headroom matters more than peak. @@ -92,7 +92,7 @@ specific version your config references — `latest` works but pinning avoids surprises. ```bash -docker pull ghcr.io/tracemachina/nativelink:v1.4.0 +docker pull ghcr.io/tracemachina/nativelink:v1.6.1 ``` The [`/pkgs/container/nativelink`](https://github.com/TraceMachina/nativelink/pkgs/container/nativelink) @@ -113,6 +113,36 @@ clients. Snapshot strategy depends on the backend: The Action Cache can be wiped without data loss; you'll re-execute everything until it warms back up. +## FAQ + + + + Yes, for permitted use. Most of the monorepo is + `FSL-1.1-Apache-2.0` — internal use, modification, and redistribution + for non-competing purposes are allowed. Metrics and remote persistent + workers are Business Source License modules: fine for an individual + developer's cache, but shared, production, or commercial use of those + modules needs NativeLink Cloud, Enterprise, or an intentionally + inexpensive commercial license. Meaningful contributors may be + eligible for waivers — contact the maintainers before relying on one. + The practical summary is on the + [license page](https://nativelink.com/license). + + + Compute for the control plane (a few small VMs), CAS storage — the + dominant cost in a healthy cluster — worker compute, and the engineer + hours to run it all. For most teams under ~30 engineers, self-hosting + is cheaper; past that, [Cloud](https://nativelink.com/pricing) + usually wins on engineer time alone. + + + No. Nothing artificially limits cache size, action count, + worker count, or team size — the self-hosted build is the same + codebase run in production, subject to each module's license, minus + the operational tooling and support the paid tiers add. + + + ## What's next - [Configuration → Production](/configuration/production) — the diff --git a/web/apps/docs/content/docs/getting-started/other-build-systems/buck2.mdx b/web/apps/docs/content/docs/getting-started/other-build-systems/buck2.mdx index f734543ca..d70d95914 100644 --- a/web/apps/docs/content/docs/getting-started/other-build-systems/buck2.mdx +++ b/web/apps/docs/content/docs/getting-started/other-build-systems/buck2.mdx @@ -91,3 +91,19 @@ download paths are working. sets `remote_enabled = True`. - **Instance-name errors.** Make sure Buck2's `instance_name` matches the NativeLink config. The integration test uses `main`. + +## FAQ + + + + `engine_address`, `action_cache_address`, and `cas_address` are + separate settings because Buck2 lets you split those services across + hosts. NativeLink serves them all from one listener, so all three + point at the same `host:port`. + + + Both. Execution needs an execution platform registered with + `remote_enabled = True` (above) plus a NativeLink scheduler and + worker — the integration test config runs one of each. + + diff --git a/web/apps/docs/content/docs/getting-started/other-build-systems/buildstream.mdx b/web/apps/docs/content/docs/getting-started/other-build-systems/buildstream.mdx index da02be24c..556f39513 100644 --- a/web/apps/docs/content/docs/getting-started/other-build-systems/buildstream.mdx +++ b/web/apps/docs/content/docs/getting-started/other-build-systems/buildstream.mdx @@ -115,3 +115,19 @@ and checks that NativeLink did not log an error. - **Remote actions do not match a worker.** Compare the platform properties BuildStream requests with the properties your NativeLink worker advertises. The integration test worker advertises `ISA: x86-64`. + +## FAQ + + + + They configure different BuildStream subsystems — artifact push/pull + storage versus the execution, action-cache, and storage services used + for remote builds. Pointing every URL at the same NativeLink listener + is fine; it exposes all of those services on one port. + + + BuildStream expresses these endpoints as URLs rather than bare + `host:port` pairs. The checked-in integration test runs without TLS, + hence `http://localhost:50051`. + + diff --git a/web/apps/docs/content/docs/getting-started/other-build-systems/cmake-recc.mdx b/web/apps/docs/content/docs/getting-started/other-build-systems/cmake-recc.mdx index a0b8731c7..b2167a1f3 100644 --- a/web/apps/docs/content/docs/getting-started/other-build-systems/cmake-recc.mdx +++ b/web/apps/docs/content/docs/getting-started/other-build-systems/cmake-recc.mdx @@ -36,19 +36,16 @@ giving you cross-machine and cross-branch caching. ## 1. Start NativeLink ```sh -curl -O https://raw.githubusercontent.com/TraceMachina/nativelink/v1.0.0/nativelink-config/examples/basic_cas.json5 +curl -O https://raw.githubusercontent.com/TraceMachina/nativelink/v1.6.1/nativelink-config/examples/basic_cas.json5 docker run -d --name nativelink \ - --platform linux/amd64 \ -v $(pwd)/basic_cas.json5:/config \ -p 50051:50051 -p 50061:50061 \ - ghcr.io/tracemachina/nativelink:v1.0.0 config + ghcr.io/tracemachina/nativelink:v1.6.1 config ``` -The prebuilt image is x86_64. `--platform linux/amd64` is a no-op on Linux -x86_64 and triggers fast emulation on Apple Silicon. (Linux arm64 users: -build NativeLink with `nix run github:TraceMachina/nativelink ./basic_cas.json5` -instead.) +The image is multi-arch (x86_64 and ARM64) as of v1.6.0, so it runs +natively on Linux and Apple Silicon alike. Confirm it came up: @@ -195,6 +192,27 @@ actions, not link actions, by default. docker stop nativelink && docker rm nativelink ``` +## FAQ + + + + Not in this setup — `RECC_CACHE_ONLY=1` means misses compile locally + and only the outputs travel through NativeLink. Drop that variable + once you have remote workers configured and recc will dispatch + compiles to them. + + + recc ships compile actions only by default; links always run locally. + For remote link steps you want a full RBE client — see + [Classic RBE examples](/rbe/examples). + + + Point `RECC_SERVER` at a shared NativeLink deployment and keep + `RECC_INSTANCE` identical across machines — mismatched instance names + produce a silently empty cache. + + + ## Going further - Point `RECC_SERVER` at a shared NativeLink deployment (cloud or diff --git a/web/apps/docs/content/docs/getting-started/other-build-systems/index.mdx b/web/apps/docs/content/docs/getting-started/other-build-systems/index.mdx index f550eb3b5..dbd1652e0 100644 --- a/web/apps/docs/content/docs/getting-started/other-build-systems/index.mdx +++ b/web/apps/docs/content/docs/getting-started/other-build-systems/index.mdx @@ -35,3 +35,27 @@ For a local Docker server, start with [Getting Started → Setup](/getting-started/setup). For the Buck2 integration test shape, see [`integration_tests/buck2/buck2_cas.json5`](https://github.com/TraceMachina/nativelink/tree/main/integration_tests/buck2/buck2_cas.json5). + +## FAQ + + + + Yes — they speak the same protocol against the same endpoint. Each + tool hashes its actions differently, so tools won't share entries + with each other, but every client of the same tool and toolchain + will. + + + Action hashes computed under different `instance_name` values never + collide, and a mismatch produces no error — just a cache that looks + permanently empty. Every client targeting a cache must send the same + instance name the server exposes. + + + If you're on CMake, + [recc in cache-only mode](/getting-started/other-build-systems/cmake-recc) + needs no worker fleet and no build-file changes. For Bazel, two + `.bazelrc` lines against the [Setup](/getting-started/setup) server + do it. + + diff --git a/web/apps/docs/content/docs/getting-started/other-build-systems/pants.mdx b/web/apps/docs/content/docs/getting-started/other-build-systems/pants.mdx index 033ae8e69..e1da90fb0 100644 --- a/web/apps/docs/content/docs/getting-started/other-build-systems/pants.mdx +++ b/web/apps/docs/content/docs/getting-started/other-build-systems/pants.mdx @@ -55,3 +55,19 @@ and toolchains aligned. config. The local examples use `main`. - **Connection failures.** Pants expects a `grpc://` URL in `remote_store_address`. + +## FAQ + + + + To let you split roles: CI can populate the cache + (`remote_cache_write = true`) while developer laptops only read from + it. For a single shared setup, enable both. + + + Yes — it's the same Remote Execution API — but the worker environment + must match what Pants sends. Get remote caching working first, then + align worker platform properties and toolchains before enabling + execution. + + diff --git a/web/apps/docs/content/docs/getting-started/other-build-systems/siso.mdx b/web/apps/docs/content/docs/getting-started/other-build-systems/siso.mdx index 2fa2f38ab..93b08200e 100644 --- a/web/apps/docs/content/docs/getting-started/other-build-systems/siso.mdx +++ b/web/apps/docs/content/docs/getting-started/other-build-systems/siso.mdx @@ -66,3 +66,18 @@ On a warm cache, cache hits should climb and network errors should stay at zero. If actions fall back to local execution, compare the platform properties Siso requests with the properties your NativeLink workers advertise. + +## FAQ + + + + Only for TLS-free local development. Against any shared endpoint, use + the TLS variables shown above instead. + + + Siso replaces Ninja and speaks full remote execution — it's the path + for Chromium-style builds. recc is a compiler launcher for ordinary + CMake projects that want caching with minimal setup — see + [CMake with recc](/getting-started/other-build-systems/cmake-recc). + + diff --git a/web/apps/docs/content/docs/getting-started/setup.mdx b/web/apps/docs/content/docs/getting-started/setup.mdx index dc5250d9e..66c0f6b0f 100644 --- a/web/apps/docs/content/docs/getting-started/setup.mdx +++ b/web/apps/docs/content/docs/getting-started/setup.mdx @@ -18,22 +18,24 @@ system to the running cluster. The fastest way is the prebuilt container image. It runs anywhere Docker runs and ships with a tested configuration. - + - + ```bash # Grab a known-good basic configuration -curl -O https://raw.githubusercontent.com/TraceMachina/nativelink/v1.4.0/nativelink-config/examples/basic_cas.json5 +curl -O https://raw.githubusercontent.com/TraceMachina/nativelink/v1.6.1/nativelink-config/examples/basic_cas.json5 # Run the official image docker run \ -v $(pwd)/basic_cas.json5:/config \ -p 50051:50051 \ - ghcr.io/tracemachina/nativelink:v1.4.0 config + ghcr.io/tracemachina/nativelink:v1.6.1 config ``` -The server is now listening on `localhost:50051`. +The image is multi-arch — x86_64 and ARM64 both run natively (use +v1.6.0 or later; earlier tags are x86_64-only). The server is now +listening on `localhost:50051`. @@ -46,8 +48,9 @@ nix run github:TraceMachina/nativelink ./basic_cas.json5 ``` Slower than the prebuilt image because it builds from source, but works on -macOS (Apple Silicon and Intel) and any Linux with Nix. Also the only path -that supports Apple Silicon natively at the moment. +macOS (Apple Silicon and Intel) and any Linux with Nix. Use it when you +want a native binary instead of a container — the Docker path above also +runs natively on Apple Silicon. - **Prerequisites**: Make sure your Nix installation has experimental features enabled (add `experimental-features = nix-command flakes` to your `~/.config/nix/nix.conf`). @@ -60,21 +63,21 @@ that supports Apple Silicon natively at the moment. - + ```powershell Invoke-WebRequest ` - -Uri "https://raw.githubusercontent.com/TraceMachina/nativelink/v1.4.0/nativelink-config/examples/basic_cas.json5" ` + -Uri "https://raw.githubusercontent.com/TraceMachina/nativelink/v1.6.1/nativelink-config/examples/basic_cas.json5" ` -OutFile "basic_cas.json5" docker run ` -v ${PWD}/basic_cas.json5:/config ` -p 50051:50051 ` - ghcr.io/tracemachina/nativelink:v1.4.0 config + ghcr.io/tracemachina/nativelink:v1.6.1 config ``` -Native Windows support is x86_64 only. ARM64 Windows users should use -WSL2 with the Linux instructions above. +The multi-arch image covers x86_64 and ARM64 Windows machines alike. +WSL2 with the Linux instructions above works too. @@ -211,6 +214,31 @@ section below. `-v $(pwd)/basic_cas.json5:/config:Z`. +## FAQ + + + + Every action's inputs — source digests, the compiler binary, the + exact command line, declared headers — are hashed together, and the + action's outputs are stored under that hash. Anyone whose action + hashes identically gets the stored outputs back in milliseconds + instead of redoing the work. + + + The hash covers every declared input, so any difference — a header, + a compiler patch version, a leaked environment variable — produces a + new hash. That's by design: a hit is a guarantee the result is + identical. If you see misses where you expect hits, start with the + [troubleshooting](#troubleshooting) section above and the + [hermeticity FAQ](/explanations/lre#faq). + + + Yes — set both, even to the same address. Without `--remote_cache`, + Bazel calls `Execute` with an action digest it never uploaded, and + the server rejects it with `Action ... is missing from CAS`. + + + ## What's next You have a local cache running. The natural next steps: diff --git a/web/apps/docs/content/docs/index.mdx b/web/apps/docs/content/docs/index.mdx index e62b50b26..a27d19973 100644 --- a/web/apps/docs/content/docs/index.mdx +++ b/web/apps/docs/content/docs/index.mdx @@ -15,7 +15,7 @@ section landings are accurate, page bodies are coming. If you're integrating NativeLink with an AI coding agent (Claude Code, Cursor, Copilot Workspace, Devin), start with [Getting Started → Setup](/getting-started/setup) - and then read [Why hermeticity matters](/faq/hermeticity). + and then read [why hermeticity matters](/explanations/lre#faq). ## Pick a starting point @@ -53,10 +53,6 @@ serves its job well. - [History](/explanations/history) — why NativeLink exists and what came before. -### I have a specific question - -- [FAQ](/faq/cost) — short answers to the questions we get most. - ### I want to contribute - [Contribution guidelines](/contribute/guidelines) — what we accept, @@ -70,3 +66,33 @@ serves its job well. configurable knob, autogenerated from the source. - [Glossary](/reference/glossary) - [Changelog](/reference/changelog) + +## FAQ + +Every page on this site ends with an FAQ covering the questions we +get most about that topic. The ones we get most overall: + + + + Yes — NativeLink is source-available and free for permitted + self-hosted use. Most of the monorepo is licensed under + `FSL-1.1-Apache-2.0`; a few modules (metrics, remote persistent + workers) are under the Business Source License, and paid + [Cloud and Enterprise](https://nativelink.com/pricing) tiers exist if + you'd rather not run it yourself. The licensing details are in + [NativeLink on-prem](/getting-started/on-prem#faq). + + + Any client that speaks the Remote Execution API: Bazel, Buck2, Siso, + Pants, BuildStream, Goma, and plain CMake via + [recc](/getting-started/other-build-systems/cmake-recc) — with no + build-file rewrites. See + [Other build systems](/getting-started/other-build-systems). + + + Linux and Windows on x86_64 and ARM64 via the multi-arch Docker image + (`ghcr.io/tracemachina/nativelink`, v1.6.0+), and macOS (Apple + Silicon and Intel) via the Docker image or a from-source Nix build. + See [Setup](/getting-started/setup). + + diff --git a/web/apps/docs/content/docs/meta.json b/web/apps/docs/content/docs/meta.json index 3b845589d..846405094 100644 --- a/web/apps/docs/content/docs/meta.json +++ b/web/apps/docs/content/docs/meta.json @@ -8,7 +8,6 @@ "deployment", "---Learn---", "explanations", - "faq", "---Develop---", "contribute", "reference" diff --git a/web/apps/docs/content/docs/rbe/examples.mdx b/web/apps/docs/content/docs/rbe/examples.mdx index 31666a1ea..f4403d273 100644 --- a/web/apps/docs/content/docs/rbe/examples.mdx +++ b/web/apps/docs/content/docs/rbe/examples.mdx @@ -113,6 +113,30 @@ Start with cache-only. Move to full RE when the team feels the local build CPU bottleneck. Move to hybrid only when you have telemetry showing where time goes. +## FAQ + + + + The action: input digests, the command line, and the platform + requirements. The worker fetches inputs from the CAS by hash, runs + the command, and uploads outputs the same way — nothing else crosses + the wire. + + + Throughput (a fleet has more cores than any laptop), hermeticity (one + job, one environment, no stale state), and cost — build hardware goes + from the 5-10% utilization of idle CI runners to 60-80% on a shared + fleet. + + + Three levels of rigor: a Nix flake (fully hermetic — the + [LRE](/explanations/lre) path), Bazel's `rules_cc` / `rules_rust` + pinned in `MODULE.bazel` (mostly hermetic), or whatever's on `$PATH` + (quick, no guarantees). Either way, the scheduler routes actions to + matching workers via `platform_properties`. + + + ## What's next - [Nix templates](/rbe/nix-templates) — runnable testbeds for diff --git a/web/apps/docs/content/docs/rbe/local-remote-execution.mdx b/web/apps/docs/content/docs/rbe/local-remote-execution.mdx index ccc194eec..2a7afd755 100644 --- a/web/apps/docs/content/docs/rbe/local-remote-execution.mdx +++ b/web/apps/docs/content/docs/rbe/local-remote-execution.mdx @@ -179,6 +179,24 @@ the same CAS your local Bazel reads from. Re-run and it's a cache hit. — it also covers verifying the setup against a Kubernetes cluster. +## FAQ + + + + [Local cache and executor](/rbe/local-testing) validates the protocol + with whatever toolchain is on your `PATH` — it has no toolchain + opinion. This page pins the toolchain with Nix so local and remote + actions hash identically. Run that one first if you just want to see + `Execute` work. + + + No — the guarantee *is* the Nix pinning: every tool lives at a + content-addressed `/nix/store` path, which is what makes action + hashes match across machines. The server side, by contrast, runs fine + without Nix. + + + ## What's next - [Explanations → LRE](/explanations/lre) — the conceptual write-up. diff --git a/web/apps/docs/content/docs/rbe/local-testing.mdx b/web/apps/docs/content/docs/rbe/local-testing.mdx index 28e4dccf9..69d96d656 100644 --- a/web/apps/docs/content/docs/rbe/local-testing.mdx +++ b/web/apps/docs/content/docs/rbe/local-testing.mdx @@ -192,6 +192,22 @@ Both runs were captured against the exact config above, on macOS platform makes the worker exit immediately. +## FAQ + + + + Yes — the config exposes standard RE-API services, so Buck2, Pants, + Siso, or recc can point at `127.0.0.1:50051` the same way. The Bazel + flags are just the worked example. + + + The process counts in Bazel's output (`3 remote`, + `2 remote cache hit`) are the quick check; + `--execution_log_json_file` records the runner per action when you + need the per-action truth. + + + ## What's next - [Local Remote Execution](/rbe/local-remote-execution) — add diff --git a/web/apps/docs/content/docs/rbe/nix-templates.mdx b/web/apps/docs/content/docs/rbe/nix-templates.mdx index 0e2dcf8e5..9c754f3ff 100644 --- a/web/apps/docs/content/docs/rbe/nix-templates.mdx +++ b/web/apps/docs/content/docs/rbe/nix-templates.mdx @@ -107,6 +107,23 @@ modifications: `flake.nix`, kept in sync with the `local-remote-execution` module override in `MODULE.bazel`). +## FAQ + + + + A package manager that builds everything in isolation and addresses + results by a hash of all inputs — your `clang` lives at + `/nix/store/-clang-.../bin/clang`. That hash-pinned path is + exactly what LRE needs to make action hashes identical across + machines. + + + A learning curve (an unfamiliar language, terse error messages) and + the first `nix develop`, which downloads the full toolchain — about + 5 GB for this template. After that, entering the shell is instant. + + + ## What's next - [Local Remote Execution](/rbe/local-remote-execution) — this same diff --git a/web/apps/docs/content/docs/reference/glossary.mdx b/web/apps/docs/content/docs/reference/glossary.mdx index cfdb455c2..2278f07c7 100644 --- a/web/apps/docs/content/docs/reference/glossary.mdx +++ b/web/apps/docs/content/docs/reference/glossary.mdx @@ -34,7 +34,7 @@ A content hash plus a size. Used everywhere instead of file paths. A build whose outputs depend only on its declared inputs. Same inputs → same outputs, on any machine, any time. See -[How do I make my Bazel setup hermetic?](/faq/hermeticity). +[the hermeticity FAQ](/explanations/lre#faq). ## Instance name @@ -81,4 +81,5 @@ like `bwrap`, `landlock`, or `sandbox-exec`. ## Toolchain The bundle of binaries an action needs to run — compiler, linker, -standard library, etc. See [What are toolchains?](/faq/toolchains). +standard library, etc. See +[how toolchains are provided](/rbe/examples#faq). diff --git a/web/apps/docs/content/docs/reference/nativelink-config/store-overview.mdx b/web/apps/docs/content/docs/reference/nativelink-config/store-overview.mdx index ca4b343fa..81637daf1 100644 --- a/web/apps/docs/content/docs/reference/nativelink-config/store-overview.mdx +++ b/web/apps/docs/content/docs/reference/nativelink-config/store-overview.mdx @@ -161,6 +161,26 @@ instead of declaring the same backend twice: } ``` +## FAQ + + + + No — it never checks that an object in `fast` also made it to `slow`. + If an artifact must survive a fast-tier wipe, make sure the write + path covers both tiers deliberately. + + + Redis caps uploads at roughly 256-512 MB. Route large blobs to a + disk- or object-store backend with `size_partitioning` and keep Redis + serving the small, hot objects it's good at. + + + `existence_cache` and `size_partitioning` are CAS-only; + `completeness_checking` is AC-only. Misplacing one isn't a config + error — it surfaces as a confusing correctness bug. + + + ## What's next - [Configuration reference](/reference/nativelink-config) — every diff --git a/web/apps/docs/mdx-components.tsx b/web/apps/docs/mdx-components.tsx index 39873cc62..1c48b6e67 100644 --- a/web/apps/docs/mdx-components.tsx +++ b/web/apps/docs/mdx-components.tsx @@ -2,6 +2,7 @@ import { Callout } from "@/components/callout"; import { ConfigVersionSwitcher } from "@/components/config-version-switcher"; import { Mermaid } from "@/components/mermaid"; import { Steps } from "@/components/steps"; +import { Accordion, Accordions } from "fumadocs-ui/components/accordion"; import { Callout as FumadocsCallout } from "fumadocs-ui/components/callout"; import { Tab, Tabs } from "fumadocs-ui/components/tabs"; import defaultMdxComponents from "fumadocs-ui/mdx"; @@ -20,6 +21,8 @@ export function getMDXComponents(components?: MDXComponents): MDXComponents { Tabs, Tab, Mermaid, + Accordions, + Accordion, ...components, }; } diff --git a/web/apps/docs/next.config.mjs b/web/apps/docs/next.config.mjs index 2fa43e350..27918d7c2 100644 --- a/web/apps/docs/next.config.mjs +++ b/web/apps/docs/next.config.mjs @@ -20,6 +20,30 @@ const nextConfig = { destination: "/getting-started/other-build-systems/siso", permanent: true, }, + // The standalone /faq section was dissolved into per-page FAQ + // sections; keep its published URLs resolving to the new homes. + ...Object.entries({ + architecture: "/explanations/architecture#faq", + caching: "/getting-started/setup#faq", + clients: "/getting-started/other-build-systems#faq", + configuration: "/configuration/intro#faq", + contributing: "/contribute/guidelines#faq", + cost: "/getting-started/on-prem#faq", + deployment: "/deployment/on-prem-overview#faq", + hermeticity: "/explanations/lre#faq", + lre: "/explanations/lre#faq", + nix: "/contribute/nix#faq", + observability: "/deployment/metrics#faq", + "remote-execution": "/rbe/examples#faq", + rust: "/explanations/history#faq", + stores: "/reference/nativelink-config/store-overview#faq", + toolchains: "/rbe/examples#faq", + troubleshooting: "/getting-started/setup#faq", + }).map(([slug, destination]) => ({ + source: `/faq/${slug}`, + destination, + permanent: true, + })), ]; }, }; From 97b961aac277dd27e0472930a3b1b4733467b8f3 Mon Sep 17 00:00:00 2001 From: Ernesto Cambuston Date: Wed, 22 Jul 2026 04:10:26 -0700 Subject: [PATCH 33/84] Coalesce FilesystemStore durability flushes on macOS (#2539) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Every FilesystemStore upload makes its blob durable with File::sync_all before publishing it, which on macOS issues fcntl(F_FULLFSYNC) — a full device-cache flush — once per blob. The flush serializes at the device and costs multiple milliseconds, so many-small-blob workloads spend nearly all their time flushing: a 4,400-file / 2 MB action input tree takes 14.5s cold (21.0s under contention), ~12s of which is F_FULLFSYNC, while tar -x of the same tree takes ~1.4s. Linux does not suffer from this because concurrent fsyncs coalesce in the filesystem journal's group commit; macOS has no journal-level coalescing and no syncfs(2), and F_BARRIERFSYNC measures just as slow per file (~15.6s). Apply the same group-commit idea in userspace, macOS-only, no config: F_FULLFSYNC's per-file part (pushing the file's pages to the device) is cheap plain fsync(2); the expensive device-cache drain is device-wide by nature. Each writer fsync(2)s its own blob (bounded by the crate's open-file permit machinery), then joins the current commit round; a single long-lived flusher task issues one F_FULLFSYNC per round on a dedicated sentinel file (a sibling of the content path, invisible to the startup scan) covering every previously-fsynced blob at once. Rounds self-batch like a journal — while one flush runs, later writers accumulate into the next round — plus a ~3ms accumulation window applied only when a round has multiple waiters, so a solo write pays about what it pays today. Durability is unchanged: a blob is only renamed into the content directory after a device-cache flush that started after its own data reached the device — the same guarantee as per-file F_FULLFSYNC. EINTR is retried exactly as std's sync_all does; the blocking fsync task owns a dup'd fd so caller cancellation can never touch a reused descriptor; a cancelled flusher round reports an error to its waiters rather than stranding them; and a store whose sentinel cannot be created (read-only volume) degrades to per-file sync_all instead of failing construction. The executable-variant copy path drops its flush entirely: the .exec directory is cleared on every startup, so those files are never trusted across a crash. Linux and Windows keep plain sync_all, byte-identical behavior. Measured on the benchmark above (output trees byte-verified every run): 14.5s -> ~3.0s alone and 21.0s -> ~5.6s contended (4.8x / 3.7x), within ~35% of the no-flush floor (2.2s / 4.1s). --- nativelink-store/BUILD.bazel | 5 +- nativelink-store/Cargo.toml | 3 + nativelink-store/src/filesystem_store.rs | 349 +++++++++++++++++- .../tests/filesystem_store_test.rs | 57 +++ 4 files changed, 402 insertions(+), 12 deletions(-) diff --git a/nativelink-store/BUILD.bazel b/nativelink-store/BUILD.bazel index 0751658fb..dffb64ea7 100644 --- a/nativelink-store/BUILD.bazel +++ b/nativelink-store/BUILD.bazel @@ -111,7 +111,10 @@ rust_library( "@crates//:uuid", "@crates//:webpki-roots", "@crates//:wincode", - ], + ] + select({ + "@platforms//os:macos": ["@crates//:libc"], + "//conditions:default": [], + }), ) rust_test_suite( diff --git a/nativelink-store/Cargo.toml b/nativelink-store/Cargo.toml index a276d2070..5f718376c 100644 --- a/nativelink-store/Cargo.toml +++ b/nativelink-store/Cargo.toml @@ -131,6 +131,9 @@ wincode = { version = "0.5.4", default-features = false, features = [ "derive", ] } +[target.'cfg(target_os = "macos")'.dependencies] +libc = { version = "0.2.177", default-features = false } + [dev-dependencies] nativelink-macro = { path = "../nativelink-macro" } diff --git a/nativelink-store/src/filesystem_store.rs b/nativelink-store/src/filesystem_store.rs index 94220e179..a21e2e96a 100644 --- a/nativelink-store/src/filesystem_store.rs +++ b/nativelink-store/src/filesystem_store.rs @@ -378,6 +378,227 @@ pub fn make_temp_key(key: &StoreKey) -> StoreKey<'static> { StoreKey::Digest(make_temp_digest(key.borrow().into_digest())) } +/// Group-commit flush coalescer (macOS only). +/// +/// On macOS, `File::sync_all` issues `fcntl(F_FULLFSYNC)` — a full +/// device-cache flush — once per call. The flush is serialized at the +/// device, costs multiple milliseconds, and dominates uploads of many +/// small blobs (a 4,400-tiny-file action input tree spends ~12s of its +/// ~14.5s materialization in these flushes). Linux does not have this +/// problem because concurrent `fsync` calls coalesce inside the +/// filesystem journal's group commit. +/// +/// This applies the same group-commit idea in userspace, exploiting an +/// asymmetry in `F_FULLFSYNC`: writing a file's pages to the storage +/// device is per-file work (plain `fsync(2)`, cheap), while the expensive +/// device-cache drain is device-wide by nature. Each writer first pushes +/// its own data to the device with `fsync(2)`, then joins the current +/// commit round; a single long-lived flusher task per DEVICE issues one +/// `F_FULLFSYNC` (on a dedicated sentinel file) per round, covering every +/// previously-`fsync`ed blob at once. Rounds self-batch exactly like a +/// journal: while one flush is running, later writers accumulate into +/// the next round. +/// +/// The coalescer (and its flusher task) is per store instance. Sharing +/// one coalescer across all stores on a device would amortize further — +/// the flush is device-wide — but a process-global flusher task cannot +/// safely outlive the tokio runtime that spawned it (multiple runtimes +/// coexist in one process, e.g. one per test); doing this correctly +/// needs a runtime-agnostic flusher (a dedicated OS thread) and is left +/// as a follow-up. +/// +/// Durability is identical to per-file `F_FULLFSYNC`: a writer only +/// proceeds (and only renames its blob into the content directory) after +/// a device-cache flush that started after its own `fsync(2)` completed. +#[cfg(target_os = "macos")] +#[derive(Debug)] +struct FlushCoalescer { + /// The round currently accepting waiters. Swapped out by the flusher + /// right before it flushes, so writers whose `fsync(2)` finished + /// after the flush began land in the next round. + current_round: parking_lot::Mutex>, + /// Wakes the flusher task. Writers notify after subscribing to their + /// round, so a wakeup can never be observed before its waiter. + wake: Arc, + /// Dedicated file the `F_FULLFSYNC` is issued on. A SIBLING of the + /// store's content path (like the `.exec` variant directory), so + /// neither the startup scan nor `move_old_cache`'s legacy root sweep + /// ever sees it. + sentinel: Arc, +} + +#[cfg(target_os = "macos")] +#[derive(Debug)] +struct FlushRound { + /// Broadcasts the flush outcome to every writer in the round. + result_tx: tokio::sync::watch::Sender>>, +} + +#[cfg(target_os = "macos")] +impl FlushRound { + fn new() -> Arc { + let (result_tx, _) = tokio::sync::watch::channel(None); + Arc::new(Self { result_tx }) + } +} + +/// Guarantees a [`FlushRound`]'s waiters always receive a result: if the +/// flusher task is cancelled at an await point (runtime shutdown), waiters +/// must get an error rather than pend forever — their own `Arc` +/// keeps the channel alive, so a dropped-sender wakeup can never happen. +#[cfg(target_os = "macos")] +struct SendOnDrop(Option>); + +#[cfg(target_os = "macos")] +impl SendOnDrop { + fn finish(mut self, result: Result<(), Error>) { + if let Some(round) = self.0.take() { + // Ignore send errors: every waiter may have been cancelled. + drop(round.result_tx.send(Some(result))); + } + } +} + +#[cfg(target_os = "macos")] +impl Drop for SendOnDrop { + fn drop(&mut self) { + if let Some(round) = self.0.take() { + drop(round.result_tx.send(Some(Err(make_err!( + Code::Internal, + "Flush coalescer round task cancelled before completing" + ))))); + } + } +} + +#[cfg(target_os = "macos")] +impl Drop for FlushCoalescer { + fn drop(&mut self) { + // Wake the flusher so it observes the dead Weak and exits instead + // of parking forever. + self.wake.notify_one(); + } +} + +#[cfg(target_os = "macos")] +impl FlushCoalescer { + /// Creates the coalescer for a store and spawns its flusher task on + /// the current runtime (the same runtime the store's uploads run on). + async fn for_content_path(content_path: &str) -> Result, Error> { + // Sibling of `content_path` (never inside it): the startup scan + // and `move_old_cache` sweep everything under the content root. + let sentinel_path = format!("{content_path}.flush_sentinel"); + let sentinel = spawn_blocking!("filesystem_store_flush_sentinel_open", move || { + std::fs::OpenOptions::new() + .write(true) + .create(true) + .truncate(false) + .open(&sentinel_path) + .map_err(|e| { + make_err!( + Code::Internal, + "Failed to create flush sentinel {sentinel_path}: {e:?}" + ) + }) + }) + .await + .err_tip(|| "Failed to join flush sentinel open task")??; + + let coalescer = Arc::new(Self { + current_round: parking_lot::Mutex::new(FlushRound::new()), + wake: Arc::new(tokio::sync::Notify::new()), + sentinel: Arc::new(sentinel), + }); + + let weak = Arc::downgrade(&coalescer); + let wake = coalescer.wake.clone(); + background_spawn!("filesystem_store_flush_coalescer", async move { + loop { + wake.notified().await; + let Some(coalescer) = weak.upgrade() else { + return; + }; + coalescer.flush_one_round().await; + // Drop the strong ref before parking so the store can be + // torn down while the flusher is idle. + } + }); + Ok(coalescer) + } + + /// Waits until a device-cache flush that started after this call has + /// completed. Callers must have already `fsync(2)`ed their own file. + async fn commit(&self) -> Result<(), Error> { + let round = self.current_round.lock().clone(); + let mut result_rx = round.result_tx.subscribe(); + // Notify strictly after subscribing: the flusher skips rounds + // with no receivers, so this ordering makes lost wakeups + // impossible (a permit is stored even while the flusher is busy). + self.wake.notify_one(); + let result_ref = result_rx + .wait_for(Option::is_some) + .await + .map_err(|_| make_err!(Code::Internal, "Flush coalescer round dropped"))?; + result_ref + .clone() + .unwrap_or_else(|| Err(make_err!(Code::Internal, "Flush round result missing"))) + } + + async fn flush_one_round(&self) { + let round = self.current_round.lock().clone(); + if round.result_tx.receiver_count() == 0 { + // Spurious wakeup (e.g. a permit left over from a round that + // a previous iteration already served). + return; + } + let send_guard = SendOnDrop(Some(round.clone())); + + // Brief accumulation window, only when this round actually has + // multiple waiters: lets a burst pack more writers into the round + // before it closes, trading ~3ms of publish latency (about the + // cost of one device flush) for fewer device flushes. A solo + // writer on an idle store skips it and pays only its own flush. + if round.result_tx.receiver_count() > 1 { + tokio::time::sleep(Duration::from_millis(3)).await; + } + // Close the round: writers arriving from here on cannot assume + // this flush covers them, so they must get a fresh round. + { + let mut current = self.current_round.lock(); + if Arc::ptr_eq(&*current, &round) { + *current = FlushRound::new(); + } + } + let sentinel = self.sentinel.clone(); + let flush_result = spawn_blocking!("filesystem_store_full_flush", move || { + use std::os::fd::AsRawFd; + use std::os::unix::fs::FileExt; + // Keep the sentinel dirty so the flush is never a no-op. + // Best-effort: a failed write (e.g. ENOSPC on a full CoW + // volume) must not fail the whole round — the fcntl below is + // what provides the device-cache drain. + if let Err(err) = sentinel.write_at(b"f", 0) { + warn!(?err, "Flush sentinel write failed; issuing flush anyway"); + } + loop { + if unsafe { libc::fcntl(sentinel.as_raw_fd(), libc::F_FULLFSYNC) } != -1 { + return Ok(()); + } + let err = std::io::Error::last_os_error(); + // std's sync_all retries EINTR (cvt_r); match it. + if err.kind() != std::io::ErrorKind::Interrupted { + return Err(Error::from(err).append("F_FULLFSYNC failed in flush coalescer")); + } + } + }) + .await + .unwrap_or_else(|e| { + Err(Error::from_std_err(Code::Internal, &e).append("Flush coalescer task failed")) + }); + send_guard.finish(flush_result); + } +} + impl LenEntry for FileEntryImpl { #[inline] fn len(&self) -> u64 { @@ -846,6 +1067,13 @@ pub struct FilesystemStore { rename_fn: fn(&OsStr, &OsStr) -> Result<(), std::io::Error>, /// Limits concurrent write operations to prevent disk I/O saturation. write_semaphore: Option, + /// See [`FlushCoalescer`]: amortizes the per-blob `F_FULLFSYNC` cost + /// across concurrent uploads without weakening durability. `None` when + /// the sentinel could not be created (e.g. read-only content volume); + /// uploads then fall back to per-file `sync_all`, matching the old + /// behavior (and such stores fail at write time anyway). + #[cfg(target_os = "macos")] + flush_coalescer: Option>, /// Per-digest single-flight locks guarding creation of the executable /// variant in `{content_path}.exec`, so each variant's writable fd is /// opened exactly once. The outer lock is sync and only ever held to @@ -892,7 +1120,16 @@ impl FilesystemStore { #[cfg(unix)] { let executable_dir = format!("{}{EXECUTABLE_DIR_SUFFIX}", spec.content_path); - drop(fs::remove_dir_all(&executable_dir).await); + // The wipe must not fail silently: a variant surviving an + // unclean shutdown would be served by the hardlink fast path, + // so a failed clear has to abort construction rather than + // risk publishing stale (or torn) executables. + if let Err(err) = fs::remove_dir_all(&executable_dir).await + && err.code != Code::NotFound + { + return Err(err) + .err_tip(|| format!("Failed to clear executable dir {executable_dir}")); + } fs::create_dir_all(format!("{executable_dir}/{DIGEST_FOLDER}")) .await .err_tip(|| format!("Failed to create executable dir {executable_dir}"))?; @@ -934,6 +1171,20 @@ impl FilesystemStore { } else { None }; + // Never make construction fail over the durability optimization: + // a read-only content volume must still serve reads, exactly as + // it did before the coalescer existed. + #[cfg(target_os = "macos")] + let flush_coalescer = FlushCoalescer::for_content_path(&shared_context.content_path) + .await + .inspect_err(|err| { + warn!( + ?err, + "Failed to create flush coalescer; falling back to per-file sync_all" + ); + }) + .ok(); + Ok(Arc::new_cyclic(|weak_self| Self { shared_context, evicting_map, @@ -942,6 +1193,8 @@ impl FilesystemStore { weak_self: weak_self.clone(), rename_fn, write_semaphore, + #[cfg(target_os = "macos")] + flush_coalescer, #[cfg(unix)] executable_locks: std::sync::Mutex::new(HashMap::new()), })) @@ -1099,10 +1352,33 @@ impl FilesystemStore { "executable-variant chmod 0o555 failed: {e:?}" ) })?; - // Reopen read-only purely to fsync the bytes durable before publish. + // Belt-and-suspenders flush before publish. The `.exec` + // directory is cleared on every startup, so durability is + // only load-bearing if that best-effort wipe ever fails — + // but flushing keeps a surviving variant VALID rather than + // possibly torn. Non-macOS keeps the prior `sync_all` + // behavior; macOS uses plain `fsync(2)` (data handed to + // the device, no multi-ms `F_FULLFSYNC` device-cache + // drain), which covers kernel panics — the startup wipe + // covers power loss. let f = std::fs::File::open(&temp_owned) .map_err(|e| make_err!(Code::Internal, "executable-variant reopen: {e:?}"))?; - f.sync_all() + #[cfg(target_os = "macos")] + let flush_result = { + use std::os::fd::AsRawFd; + loop { + if unsafe { libc::fsync(f.as_raw_fd()) } == 0 { + break Ok(()); + } + let err = std::io::Error::last_os_error(); + if err.kind() != std::io::ErrorKind::Interrupted { + break Err(err); + } + } + }; + #[cfg(not(target_os = "macos"))] + let flush_result = f.sync_all(); + flush_result .map_err(|e| make_err!(Code::Internal, "executable-variant fsync: {e:?}"))?; drop(f); rename_fn(temp_owned.as_os_str(), variant_owned.as_os_str()).map_err(|e| { @@ -1175,11 +1451,9 @@ impl FilesystemStore { .flush() .await .err_tip(|| "Failed to flush in filesystem store")?; - temp_file - .as_ref() - .sync_all() + self.flush_durably(&temp_file) .await - .err_tip(|| "Failed to sync_data in filesystem store")?; + .err_tip(|| "Failed to sync in filesystem store")?; drop(permit); @@ -1302,6 +1576,61 @@ impl FilesystemStore { .err_tip(|| "Failed to create spawn in filesystem store update_file")? } + /// Makes `file`'s contents durable with `sync_all` semantics. + /// + /// On macOS the naive equivalent (`fcntl(F_FULLFSYNC)` per file) is a + /// full device-cache flush that serializes at the device and dominates + /// many-small-blob uploads, so the flush is split into the cheap + /// per-file part (`fsync(2)`: push this file's pages to the device) + /// and a device-wide cache drain shared with every concurrent upload + /// via [`FlushCoalescer`]. Other platforms get their journal's group + /// commit for free and keep plain `sync_all`. + async fn flush_durably(&self, file: &FileSlot) -> Result<(), Error> { + #[cfg(target_os = "macos")] + { + use std::os::fd::AsRawFd; + let Some(flush_coalescer) = &self.flush_coalescer else { + return file.as_ref().sync_all().await.map_err(Into::into); + }; + // Dup the handle so the blocking task owns its own fd: if + // this future is cancelled mid-await, the temp file's fd can + // close and be reused, and an already-started blocking task + // must never fsync an unrelated descriptor. + let file_dup = file + .as_ref() + .try_clone() + .await + .err_tip(|| "Failed to dup fd for fsync in filesystem store")? + .into_std() + .await; + // Deliberately NOT via `fs::call_with_permit`: the caller's + // `FileSlot` already holds an open-file permit, so requiring a + // second one here deadlocks when the semaphore is drained (the + // exact scenario the #2051 regression test pins at one + // permit). Concurrency is still bounded: every in-flight fsync + // belongs to an upload holding a `FileSlot` permit. + spawn_blocking!("filesystem_store_fsync", move || { + loop { + if unsafe { libc::fsync(file_dup.as_raw_fd()) } == 0 { + return Ok(()); + } + let err = std::io::Error::last_os_error(); + // std's sync_all retries EINTR (cvt_r); match it. + if err.kind() != std::io::ErrorKind::Interrupted { + return Err(Error::from(err).append("fsync failed in filesystem store")); + } + } + }) + .await + .err_tip(|| "Failed to join fsync task in filesystem store")??; + flush_coalescer.commit().await + } + #[cfg(not(target_os = "macos"))] + { + file.as_ref().sync_all().await.map_err(Into::into) + } + } + pub fn get_eviction_snapshot(&self) -> EvictionSnapshot { self.evicting_map.get_snapshot() } @@ -1440,11 +1769,9 @@ impl StoreDriver for FilesystemStore { .flush() .await .err_tip(|| "Failed to flush in filesystem store update_oneshot")?; - temp_file - .as_ref() - .sync_all() + self.flush_durably(&temp_file) .await - .err_tip(|| "Failed to sync_data in filesystem store update_oneshot")?; + .err_tip(|| "Failed to sync in filesystem store update_oneshot")?; drop(_permit); diff --git a/nativelink-store/tests/filesystem_store_test.rs b/nativelink-store/tests/filesystem_store_test.rs index 139a40b44..c5754615b 100644 --- a/nativelink-store/tests/filesystem_store_test.rs +++ b/nativelink-store/tests/filesystem_store_test.rs @@ -1982,3 +1982,60 @@ async fn unref_does_not_orphan_content_file_when_temp_dir_missing() -> Result<() Ok(()) } + +// Exercises the macOS flush coalescer's round machinery under a burst of +// concurrent uploads (on other platforms this is a plain concurrency +// smoke test): every upload must complete durably and round-trip, and a +// store restart must still see every blob. +#[nativelink_test(flavor = "multi_thread", worker_threads = 4)] +async fn concurrent_upload_burst_round_trip_test() -> Result<(), Error> { + const NUM_BLOBS: u64 = 200; + let content_path = make_temp_path("content_path_burst"); + let temp_path = make_temp_path("temp_path_burst"); + let spec = FilesystemSpec { + content_path: content_path.clone(), + temp_path: temp_path.clone(), + eviction_policy: None, + block_size: 1, + ..Default::default() + }; + let make_digest = |i: u64| { + let mut hash = [0u8; 32]; + hash[0] = 0xbb; + hash[1..9].copy_from_slice(&i.to_le_bytes()); + DigestInfo::new(hash, 25) + }; + { + let store = Store::new(FilesystemStore::::new(&spec).await?); + let mut handles = Vec::new(); + for i in 0..NUM_BLOBS { + let store = store.clone(); + handles.push(tokio::spawn(async move { + let content = format!("burst-content-{i:011}"); + store + .update_oneshot(make_digest(i), content.into_bytes().into()) + .await + })); + } + for handle in handles { + handle.await.expect("upload task panicked")?; + } + for i in 0..NUM_BLOBS { + assert_eq!( + store.get_part_unchunked(make_digest(i), 0, None).await?, + format!("burst-content-{i:011}").as_bytes(), + "round trip failed for blob {i}" + ); + } + } + // A fresh store over the same paths must find every published blob. + let store = Store::new(FilesystemStore::::new(&spec).await?); + for i in 0..NUM_BLOBS { + assert_eq!( + store.has(make_digest(i)).await?, + Some(25), + "blob {i} missing after restart" + ); + } + Ok(()) +} From 02e3e7665ef566da7eabb10a8d7622d6f0b9dbf3 Mon Sep 17 00:00:00 2001 From: Aman Kumar Date: Wed, 22 Jul 2026 13:20:06 +0100 Subject: [PATCH 34/84] Skip pre_exec hook when namespaces are disabled to allow posix_spawn (#2597) --- .../src/running_actions_manager.rs | 33 ++--- .../tests/running_actions_manager_test.rs | 122 ++++++++++++++++++ 2 files changed, 140 insertions(+), 15 deletions(-) diff --git a/nativelink-worker/src/running_actions_manager.rs b/nativelink-worker/src/running_actions_manager.rs index 6e4b20573..f0e5000fc 100644 --- a/nativelink-worker/src/running_actions_manager.rs +++ b/nativelink-worker/src/running_actions_manager.rs @@ -1459,23 +1459,26 @@ impl RunningActionImpl { #[cfg(target_os = "linux")] { let use_namespaces = self.running_actions_manager.use_namespaces; - let root_action_directory = - std::ffi::CString::new(self.running_actions_manager.root_action_directory.clone()) - .err_tip(|| "In RunningActionImpl::inner_execute()")?; - let action_directory = std::ffi::CString::new(self.action_directory.clone()) + + if !matches!(use_namespaces, UseNamespaces::No) { + let root_action_directory = std::ffi::CString::new( + self.running_actions_manager.root_action_directory.clone(), + ) .err_tip(|| "In RunningActionImpl::inner_execute()")?; + let action_directory = std::ffi::CString::new(self.action_directory.clone()) + .err_tip(|| "In RunningActionImpl::inner_execute()")?; - // SAFETY: This function is specifically designed to operate in a async-signal-safe - // environment. - unsafe { - command_builder.pre_exec(move || match use_namespaces { - UseNamespaces::No => Ok(()), - _ => crate::namespace_utils::configure_namespace( - matches!(use_namespaces, UseNamespaces::YesAndMount), - &root_action_directory, - &action_directory, - ), - }); + // SAFETY: This function is specifically designed to operate in a async-signal-safe + // environment. + unsafe { + command_builder.pre_exec(move || { + crate::namespace_utils::configure_namespace( + matches!(use_namespaces, UseNamespaces::YesAndMount), + &root_action_directory, + &action_directory, + ) + }); + } } // Run the action as its own process-group leader (pgid == child diff --git a/nativelink-worker/tests/running_actions_manager_test.rs b/nativelink-worker/tests/running_actions_manager_test.rs index 8768af642..c2ae7f5ba 100644 --- a/nativelink-worker/tests/running_actions_manager_test.rs +++ b/nativelink-worker/tests/running_actions_manager_test.rs @@ -3719,6 +3719,128 @@ exit 1 Ok(()) } + /// Regression for skipping the `pre_exec` hook when namespaces are off. + /// With namespaces disabled (the default) the action spawn goes through + /// `posix_spawn` instead of `fork`, and `process_group(0)` must still make + /// the child its own process-group leader (pgid == pid). + #[cfg(target_os = "linux")] + #[nativelink_test] + async fn no_namespace_action_is_process_group_leader() -> Result<(), Box> + { + const WORKER_ID: &str = "foo_worker_id"; + + fn test_monotonic_clock() -> SystemTime { + static CLOCK: AtomicU64 = AtomicU64::new(0); + monotonic_clock(&CLOCK) + } + + let (_, _, cas_store, ac_store) = setup_stores().await?; + let root_action_directory = make_temp_path("root_action_directory"); + fs::create_dir_all(&root_action_directory).await?; + + let running_actions_manager = Arc::new(RunningActionsManagerImpl::new_with_callbacks( + RunningActionsManagerArgs { + root_action_directory, + cas_store: cas_store.clone(), + ac_store: Some(Store::new(ac_store.clone())), + execution_configuration: ExecutionConfiguration::default(), + historical_store: Store::new(cas_store.clone()), + upload_action_result_config: &UploadActionResultConfig { + upload_ac_results_strategy: UploadCacheResultsStrategy::Never, + ..Default::default() + }, + max_action_timeout: Duration::MAX, + max_upload_timeout: Duration::from_secs(DEFAULT_MAX_UPLOAD_TIMEOUT), + max_cleanup_wait: Duration::from_secs(DEFAULT_MAX_CLEANUP_WAIT), + max_cleanup_backoff: Duration::from_millis(DEFAULT_MAX_CLEANUP_BACKOFF), + timeout_handled_externally: false, + directory_cache: None, + // Pin namespaces off so this exercises the no-pre_exec/posix_spawn + // path regardless of what the host kernel supports. + use_namespaces: nativelink_worker::running_actions_manager::UseNamespaces::No, + }, + Callbacks { + now_fn: test_monotonic_clock, + sleep_fn: |_duration| Box::pin(future::pending()), + }, + )?); + + // Print the shell's own pid (field 1) and process-group id (field 5) + // from its /proc stat line; process_group(0) makes them equal. + let command = Command { + arguments: vec![ + "sh".to_string(), + "-c".to_string(), + "read -r pid _ _ _ pgrp _ < /proc/$$/stat; printf '%s %s' \"$pid\" \"$pgrp\"" + .to_string(), + ], + output_paths: vec![], + working_directory: ".".to_string(), + environment_variables: vec![EnvironmentVariable { + name: "PATH".to_string(), + value: env::var("PATH").unwrap(), + }], + ..Default::default() + }; + let command_digest = serialize_and_upload_message( + &command, + cas_store.as_pin(), + &mut DigestHasherFunc::Sha256.hasher(), + ) + .await?; + let input_root_digest = serialize_and_upload_message( + &Directory::default(), + cas_store.as_pin(), + &mut DigestHasherFunc::Sha256.hasher(), + ) + .await?; + let action = Action { + command_digest: Some(command_digest.into()), + input_root_digest: Some(input_root_digest.into()), + ..Default::default() + }; + let action_digest = serialize_and_upload_message( + &action, + cas_store.as_pin(), + &mut DigestHasherFunc::Sha256.hasher(), + ) + .await?; + + let execute_request = ExecuteRequest { + action_digest: Some(action_digest.into()), + ..Default::default() + }; + let operation_id = OperationId::default().to_string(); + let running_action_impl = running_actions_manager + .create_and_add_action( + WORKER_ID.to_string(), + StartExecute { + execute_request: Some(execute_request), + operation_id, + ..Default::default() + }, + ) + .await?; + + let action_result = run_action(running_action_impl.clone()).await?; + assert_eq!( + action_result.exit_code, 0, + "action should run to completion via posix_spawn" + ); + + let stdout = cas_store + .as_ref() + .get_part_unchunked(action_result.stdout_digest, 0, None) + .await?; + let stdout = from_utf8(&stdout)?; + let (pid, pgrp) = stdout.split_once(' ').expect("expected 'pid pgrp' output"); + assert_eq!( + pid, pgrp, + "spawned process should be its own process-group leader (pid={pid} pgrp={pgrp})" + ); + Ok(()) + } + #[nativelink_test] async fn action_directory_contents_are_cleaned() -> Result<(), Box> { const WORKER_ID: &str = "foo_worker_id"; From 4e2f554f76789edf1072c97db39b8f84906e6a3e Mon Sep 17 00:00:00 2001 From: Tom Parker-Shemilt Date: Wed, 22 Jul 2026 14:33:35 +0100 Subject: [PATCH 35/84] Add trace logging for bytestream read (#2482) --- nativelink-service/src/bytestream_server.rs | 16 ++++++++++++++-- .../tests/bytestream_server_test.rs | 15 +++++++++++++++ 2 files changed, 29 insertions(+), 2 deletions(-) diff --git a/nativelink-service/src/bytestream_server.rs b/nativelink-service/src/bytestream_server.rs index c678eed59..e8ff31b1b 100644 --- a/nativelink-service/src/bytestream_server.rs +++ b/nativelink-service/src/bytestream_server.rs @@ -1363,7 +1363,8 @@ impl ByteStream for ByteStreamServer { let start_time = Instant::now(); let read_request = grpc_request.into_inner(); - let resource_info = ResourceInfo::new(&read_request.resource_name, false)?; + let resource_name = read_request.resource_name.clone(); + let resource_info = ResourceInfo::new(&resource_name, false)?; let instance_name = resource_info.instance_name.as_ref(); let expected_size = resource_info.expected_size as u64; let instance = self @@ -1371,6 +1372,11 @@ impl ByteStream for ByteStreamServer { .get(instance_name) .err_tip(|| format!("'instance_name' not configured for '{instance_name}'"))?; + trace!( + resource_name, + instance_name, expected_size, "Starting bytestream request" + ); + // Track read request instance .metrics @@ -1423,13 +1429,19 @@ impl ByteStream for ByteStreamServer { }; // Track metrics based on result + let elapsed = start_time.elapsed(); #[allow(clippy::cast_possible_truncation)] - let elapsed_ns = start_time.elapsed().as_nanos() as u64; + let elapsed_ns = elapsed.as_nanos() as u64; instance .metrics .read_duration_ns .fetch_add(elapsed_ns, Ordering::Relaxed); + trace!( + ?elapsed, + resource_name, instance_name, expected_size, "Completed bytestream request" + ); + match &resp { Ok(_) => { instance diff --git a/nativelink-service/tests/bytestream_server_test.rs b/nativelink-service/tests/bytestream_server_test.rs index e1c78b439..a5229fd36 100644 --- a/nativelink-service/tests/bytestream_server_test.rs +++ b/nativelink-service/tests/bytestream_server_test.rs @@ -1243,6 +1243,21 @@ pub async fn chunked_stream_reads_small_set_of_data() -> Result<(), Box Date: Wed, 22 Jul 2026 16:40:06 +0100 Subject: [PATCH 36/84] Fix custom-image workflow: grant pull-requests: write and fix command parsing (#2600) --- .github/workflows/custom-image.yaml | 122 ++++++++++++++++------------ 1 file changed, 70 insertions(+), 52 deletions(-) diff --git a/.github/workflows/custom-image.yaml b/.github/workflows/custom-image.yaml index 3de1ce34a..73827b220 100644 --- a/.github/workflows/custom-image.yaml +++ b/.github/workflows/custom-image.yaml @@ -29,7 +29,7 @@ jobs: runs-on: ubuntu-24.04 permissions: issues: write - pull-requests: read + pull-requests: write outputs: should_build: ${{ steps.check.outputs.should_build }} pr_sha: ${{ steps.check.outputs.pr_sha }} @@ -46,67 +46,78 @@ jobs: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd with: script: | - var fs = require('fs'); - var matrix = JSON.parse(fs.readFileSync('.github/images-matrix.json', 'utf8')); - var validImages = []; - for (const option of matrix['include']) { - validImages.push(option['image']); - } + const fs = require('fs'); + const matrix = JSON.parse(fs.readFileSync('.github/images-matrix.json', 'utf8')); + const validImages = matrix['include'].map((o) => o['image']); + + // Never fail the job over a courtesy comment (e.g. a read-only + // token); the build is what matters. + const tryComment = async (body) => { + try { + await github.rest.issues.createComment({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: context.payload.issue.number, + body, + }); + } catch (e) { + core.warning(`Could not post comment: ${e.message}`); + } + }; + + let image; + let prSha; + if (context.eventName === 'workflow_dispatch') { - core.setOutput('should_build', 'true'); - core.setOutput('pr_sha', context.sha); image = '${{ inputs.image }}'; - } - else if (context.eventName === 'issue_comment') { + prSha = context.sha; + } else if (context.eventName === 'issue_comment') { const body = context.payload.comment.body.trim(); const isPR = !!context.payload.issue.pull_request; - // Match /build-image or /build-image const match = body.match(/^\/build-image(?:\s+(\S+))?/i); - - if (isPR && match) { - const { data: pr } = await github.rest.pulls.get({ - owner: context.repo.owner, - repo: context.repo.repo, - pull_number: context.payload.issue.number - }); - - const image = match[1] || 'image'; - - if (!validImages.includes(image)) { - await github.rest.issues.createComment({ - owner: context.repo.owner, - repo: context.repo.repo, - issue_number: context.payload.issue.number, - body: `Unknown image: \`${image}\`\n\nValid options: ${validImages.map(i => `\`${i}\``).join(', ')}` - }); - core.setOutput('should_build', 'false'); - return; - } - - core.setOutput('should_build', 'true'); - core.setOutput('pr_sha', pr.head.sha); - core.setOutput('image', image); - + if (!isPR || !match) { + core.setOutput('should_build', 'false'); + return; + } + image = match[1]; + if (!image || !validImages.includes(image)) { + const hint = image + ? `Unknown image: \`${image}\`` + : 'Specify an image to build.'; + await tryComment( + `${hint}\n\nValid options: ${validImages.map((i) => `\`${i}\``).join(', ')}` + ); + core.setOutput('should_build', 'false'); + return; + } + const { data: pr } = await github.rest.pulls.get({ + owner: context.repo.owner, + repo: context.repo.repo, + pull_number: context.payload.issue.number, + }); + prSha = pr.head.sha; + try { await github.rest.reactions.createForIssueComment({ owner: context.repo.owner, repo: context.repo.repo, comment_id: context.payload.comment.id, - content: 'rocket' + content: 'rocket', }); + } catch (e) { + core.warning(`Could not add reaction: ${e.message}`); } } else { core.setOutput('should_build', 'false'); - return + return; } + + const option = matrix['include'].find((o) => o['image'] === image); + core.setOutput('should_build', 'true'); + core.setOutput('pr_sha', prSha); core.setOutput('image', image); - for (const option of matrix['include']) { - if (option['image'] == image) { - core.setOutput('multi-arch', option['multi-arch'] ?? false); - core.setOutput('components', option['components'] ?? ""); - return - } - } + core.setOutput('multi-arch', option?.['multi-arch'] ?? false); + core.setOutput('components', option?.['components'] ?? ''); build-image: name: Build and Push Image @@ -116,6 +127,7 @@ jobs: permissions: contents: read issues: write + pull-requests: write timeout-minutes: 45 steps: - name: Checkout @@ -155,14 +167,20 @@ jobs: if: github.event_name == 'issue_comment' uses: >- #v8.0.0 actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd + env: + IMAGE_REF: ghcr.io/${{ github.repository_owner }}/${{ needs.check-trigger.outputs.image }}:${{ steps.version.outputs.version-string }} with: script: | - await github.rest.issues.createComment({ - owner: context.repo.owner, - repo: context.repo.repo, - issue_number: context.payload.issue.number, - body: `Image built and pushed!\n\n\`\`\`\n${{ steps.upload.outputs.image_tag }}\n\`\`\`` - }); + try { + await github.rest.issues.createComment({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: context.payload.issue.number, + body: `Image built and pushed!\n\n\`\`\`\n${process.env.IMAGE_REF}\n\`\`\``, + }); + } catch (e) { + core.warning(`Could not post comment: ${e.message}`); + } - name: Teardown Worker uses: ./.github/actions/end-nix From 9fe34de2eb6a28b3414fa2e2016c56e657bda279 Mon Sep 17 00:00:00 2001 From: Marcus Eagan Date: Wed, 22 Jul 2026 17:48:29 +0100 Subject: [PATCH 37/84] Commit public read-only cache defaults so fork PRs actually receive them (#2604) The fork rerun of #2546 proved GitHub passes neither secrets nor repository variables into fork-triggered runs, so the variable-fed public credentials never reached forks. Move the public pair (cache_read-only key + claim base, both public by design) into the action's committed input defaults, drop the now-inert variable plumbing from the lane workflows, and let the public path activate without the ci-mode variable (forks cannot see it; revoking the key is the fork kill switch). Trusted lanes are unchanged: private secrets always win, and missing ci-mode disables rather than downgrading to the public path. --- .../setup-nativelink-cloud/action.yaml | 41 +++++++++++-------- .github/workflows/lre.yaml | 2 - .github/workflows/native-bazel.yaml | 4 -- .github/workflows/nix.yaml | 2 - .github/workflows/sanitizers.yaml | 2 - 5 files changed, 25 insertions(+), 26 deletions(-) diff --git a/.github/actions/setup-nativelink-cloud/action.yaml b/.github/actions/setup-nativelink-cloud/action.yaml index 6b5b1664f..17306a00c 100644 --- a/.github/actions/setup-nativelink-cloud/action.yaml +++ b/.github/actions/setup-nativelink-cloud/action.yaml @@ -3,10 +3,11 @@ name: Setup NativeLink Cloud description: >- Writes a user.bazelrc that opts this job into NativeLink Cloud remote cache/BES (and optionally remote execution) via the configs in ci.bazelrc. - Endpoints are derived from the claim-base input, which comes from a - repository secret, so no claim hostname lives in the repository. No-ops - when the API key or claim base is unavailable (fork PRs have no secrets) - or when ci-mode is 'off', so builds degrade to plain local execution. + Trusted lanes pass secret-fed credentials; fork PRs, which receive neither + secrets nor repository variables, fall back to the committed public + read-only defaults below (cache reads only — no uploads, no BES, no remote + execution). ci-mode 'off' disables trusted lanes; the public path's kill + switch is revoking the public key server-side. inputs: api-key: @@ -32,20 +33,20 @@ inputs: default: '' public-read-key: description: >- - World-readable cache_read-only API key, normally passed from - vars.NATIVELINK_PUBLIC_CACHE_KEY. Used only when api-key/claim-base are - unavailable (fork PRs have no secrets), giving fork contributors - read-only cache hits. Deliberately a variable rather than a secret: - fork code runs with the value in hand either way, so it is public by - definition and must never carry more than cache_read. + World-readable cache_read-only API key used when api-key/claim-base are + unavailable (fork PRs receive neither secrets nor repository variables, + so the value must live here as a committed default to reach them). + Deliberately public: fork code runs with the value in hand either way, + so it must never carry more than cache_read. Kill switch is revoking + the key server-side; rotation is editing this default. required: false - default: '' + default: '66532088547d98bfb5b0b41e8d5cbfea8b5da1bc1e23e776f9cefc12654c6cb6' public-claim-base: description: >- - Claim endpoint base for the public read-only path, normally passed from - vars.NATIVELINK_PUBLIC_CLAIM_BASE. + Claim endpoint base for the public read-only path. Committed default + for the same reason as public-read-key. required: false - default: '' + default: 'marcus-eagan-h6a62e.staging.scdev.nativelink.net' ci-mode: description: >- Kill switch, normally passed as vars.NATIVELINK_CI_MODE. Empty or 'off' @@ -99,10 +100,18 @@ runs: set -euo pipefail cred_kind="" - if [[ -n "$NL_CI_MODE" && "$NL_CI_MODE" != "off" ]]; then + if [[ "$NL_CI_MODE" != "off" ]]; then if [[ -n "$NL_API_KEY" && -n "$NL_CLAIM_BASE" ]]; then - cred_kind="private" + # Private credentials also require the kill-switch variable to be + # explicitly on; never silently downgrade a trusted lane to the + # public path. + if [[ -n "$NL_CI_MODE" ]]; then + cred_kind="private" + fi elif [[ -n "$NL_PUBLIC_KEY" && -n "$NL_PUBLIC_CLAIM_BASE" ]]; then + # Fork runs receive neither secrets nor repository variables, so + # the public read-only path cannot depend on ci-mode; revoking + # the public key server-side is its kill switch. cred_kind="public" fi fi diff --git a/.github/workflows/lre.yaml b/.github/workflows/lre.yaml index b83963a3c..a9229cc12 100644 --- a/.github/workflows/lre.yaml +++ b/.github/workflows/lre.yaml @@ -48,8 +48,6 @@ jobs: ci-mode: ${{ vars.NATIVELINK_CI_MODE }} claim-base: ${{ secrets.NATIVELINK_STAGING_CLAIM_BASE }} bes-results-url: ${{ secrets.NATIVELINK_STAGING_BES_RESULTS_URL }} - public-read-key: ${{ vars.NATIVELINK_PUBLIC_CACHE_KEY }} - public-claim-base: ${{ vars.NATIVELINK_PUBLIC_CLAIM_BASE }} api-key: >- ${{ github.event_name == 'push' && secrets.NATIVELINK_STAGING_API_KEY_RW diff --git a/.github/workflows/native-bazel.yaml b/.github/workflows/native-bazel.yaml index e4e02db87..7161dab46 100644 --- a/.github/workflows/native-bazel.yaml +++ b/.github/workflows/native-bazel.yaml @@ -66,8 +66,6 @@ jobs: ci-mode: ${{ vars.NATIVELINK_CI_MODE }} claim-base: ${{ secrets.NATIVELINK_STAGING_CLAIM_BASE }} bes-results-url: ${{ secrets.NATIVELINK_STAGING_BES_RESULTS_URL }} - public-read-key: ${{ vars.NATIVELINK_PUBLIC_CACHE_KEY }} - public-claim-base: ${{ vars.NATIVELINK_PUBLIC_CLAIM_BASE }} api-key: >- ${{ github.event_name == 'push' && secrets.NATIVELINK_STAGING_API_KEY_RW @@ -135,8 +133,6 @@ jobs: ci-mode: ${{ vars.NATIVELINK_CI_MODE }} claim-base: ${{ secrets.NATIVELINK_STAGING_CLAIM_BASE }} bes-results-url: ${{ secrets.NATIVELINK_STAGING_BES_RESULTS_URL }} - public-read-key: ${{ vars.NATIVELINK_PUBLIC_CACHE_KEY }} - public-claim-base: ${{ vars.NATIVELINK_PUBLIC_CLAIM_BASE }} api-key: >- ${{ github.event_name == 'push' && secrets.NATIVELINK_STAGING_API_KEY_RW diff --git a/.github/workflows/nix.yaml b/.github/workflows/nix.yaml index a9ad30ef5..31a7fae90 100644 --- a/.github/workflows/nix.yaml +++ b/.github/workflows/nix.yaml @@ -44,8 +44,6 @@ jobs: ci-mode: ${{ vars.NATIVELINK_CI_MODE }} claim-base: ${{ secrets.NATIVELINK_STAGING_CLAIM_BASE }} bes-results-url: ${{ secrets.NATIVELINK_STAGING_BES_RESULTS_URL }} - public-read-key: ${{ vars.NATIVELINK_PUBLIC_CACHE_KEY }} - public-claim-base: ${{ vars.NATIVELINK_PUBLIC_CLAIM_BASE }} api-key: >- ${{ github.event_name == 'push' && secrets.NATIVELINK_STAGING_API_KEY_RW diff --git a/.github/workflows/sanitizers.yaml b/.github/workflows/sanitizers.yaml index 8a1bb59bf..2eb7aa65c 100644 --- a/.github/workflows/sanitizers.yaml +++ b/.github/workflows/sanitizers.yaml @@ -50,8 +50,6 @@ jobs: ci-mode: ${{ vars.NATIVELINK_CI_MODE }} claim-base: ${{ secrets.NATIVELINK_STAGING_CLAIM_BASE }} bes-results-url: ${{ secrets.NATIVELINK_STAGING_BES_RESULTS_URL }} - public-read-key: ${{ vars.NATIVELINK_PUBLIC_CACHE_KEY }} - public-claim-base: ${{ vars.NATIVELINK_PUBLIC_CLAIM_BASE }} api-key: >- ${{ github.event_name == 'push' && secrets.NATIVELINK_STAGING_API_KEY_RW From 9700a4a522de525b9121188f19bd1c74b2d4a10e Mon Sep 17 00:00:00 2001 From: Alec Maliwanag Date: Wed, 22 Jul 2026 18:51:57 +0100 Subject: [PATCH 38/84] Add the Leadfeeder tracker to the marketing site (#2606) nativelink.com stopped reporting to Leadfeeder when the old Astro site (web/platform) was removed in #2371; the tracker originally added in #1474 was never carried over to the Next.js app. Restore the vendor snippet with the same site ID used on tracemachina.com, loaded via next/script afterInteractive, with preconnect hints for the script and beacon hosts. --- web/apps/web/app/layout.tsx | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/web/apps/web/app/layout.tsx b/web/apps/web/app/layout.tsx index 8207c6625..625b175f6 100644 --- a/web/apps/web/app/layout.tsx +++ b/web/apps/web/app/layout.tsx @@ -2,9 +2,14 @@ import { FinalCTA, SiteFooter, SiteHeader, ThemeProvider, themeInitScript } from import { GeistMono } from "geist/font/mono"; import { GeistSans } from "geist/font/sans"; import type { Metadata } from "next"; +import Script from "next/script"; import type { ReactNode } from "react"; import "./globals.css"; +// Official Leadfeeder (Dealfront) tracker snippet, verbatim. The site ID is the +// same one deployed on tracemachina.com. +const leadfeederScript = `(function(ss,ex){ window.ldfdr=window.ldfdr||function(){(ldfdr._q=ldfdr._q||[]).push([].slice.call(arguments));}; (function(d,s){ fs=d.getElementsByTagName(s)[0]; function ce(src){ var cs=d.createElement(s); cs.src=src; cs.async=1; fs.parentNode.insertBefore(cs,fs); }; ce('https://sc.lfeeder.com/lftracker_v1_'+ss+(ex?'_'+ex:'')+'.js'); })(document,'script'); })('lAxoEaKMQGd7OYGd');`; + export const metadata: Metadata = { title: { default: "NativeLink — Remote build execution & caching", @@ -24,6 +29,11 @@ export default function RootLayout({ children }: { children: ReactNode }) { className={`${GeistSans.variable} ${GeistMono.variable}`} > + {/* Leadfeeder origins: script host and beacon host. No crossorigin — + the tracker loads as a plain script, which can't reuse a CORS-warmed + connection. */} + + {/* biome-ignore lint/security/noDangerouslySetInnerHtml: Inline before hydration so the theme is correct on first paint. */} ); From 94f312a043847d5dd0f8ca95b676b00d4697c04d Mon Sep 17 00:00:00 2001 From: Alec Maliwanag Date: Thu, 23 Jul 2026 10:40:12 +0100 Subject: [PATCH 39/84] Add Google Tag Manager to the marketing site (#2610) The GTM container GTM-NNLLRWGB was loaded on the old Astro site (web/platform) and lost when #2371 removed it. The old loader also had an interpolation bug that left its gtag config call referencing an undefined variable. Restore the container with the canonical gtm.js install via Next's first-party GoogleTagManager component, loaded after hydration off the critical path. --- web/apps/web/app/layout.tsx | 2 ++ web/apps/web/package.json | 1 + web/bun.lock | 6 ++++++ 3 files changed, 9 insertions(+) diff --git a/web/apps/web/app/layout.tsx b/web/apps/web/app/layout.tsx index 625b175f6..33acfa58e 100644 --- a/web/apps/web/app/layout.tsx +++ b/web/apps/web/app/layout.tsx @@ -1,4 +1,5 @@ import { FinalCTA, SiteFooter, SiteHeader, ThemeProvider, themeInitScript } from "@nativelink/ui"; +import { GoogleTagManager } from "@next/third-parties/google"; import { GeistMono } from "geist/font/mono"; import { GeistSans } from "geist/font/sans"; import type { Metadata } from "next"; @@ -37,6 +38,7 @@ export default function RootLayout({ children }: { children: ReactNode }) { {/* biome-ignore lint/security/noDangerouslySetInnerHtml: Inline before hydration so the theme is correct on first paint. */} ); From 66055ac403391cdeeebcd45204b8b45e1c62171e Mon Sep 17 00:00:00 2001 From: Tom Parker-Shemilt Date: Mon, 27 Jul 2026 14:06:49 +0100 Subject: [PATCH 59/84] Move Github bazel retry into a general command (#2616) --- .github/workflows/lre.yaml | 2 +- .github/workflows/nix.yaml | 29 +++------------------- flake.nix | 25 +++++++++++++++++++ templates/bazel/.github/workflows/lre.yaml | 2 +- 4 files changed, 31 insertions(+), 27 deletions(-) diff --git a/.github/workflows/lre.yaml b/.github/workflows/lre.yaml index a9229cc12..9831e6ed9 100644 --- a/.github/workflows/lre.yaml +++ b/.github/workflows/lre.yaml @@ -59,7 +59,7 @@ jobs: TOOLCHAIN: ${{ matrix.toolchain }} run: > nix develop --impure --command - bash -c "bazel run \ + bash -c "bazel-retry run \ --lockfile_mode=error \ --verbose_failures \ @local-remote-execution//examples:${TOOLCHAIN}" diff --git a/.github/workflows/nix.yaml b/.github/workflows/nix.yaml index 31a7fae90..378b9bff4 100644 --- a/.github/workflows/nix.yaml +++ b/.github/workflows/nix.yaml @@ -52,31 +52,10 @@ jobs: - name: Invoke Bazel build in Nix shell run: | - set -o pipefail - - # Bazel's downloader retry only covers truncated downloads, not HTTP 5xx/403 failures, - # so a flaky GitHub fetch aborts the build with no retry. We retry the download in this case. - delay=5 - for attempt in 1 2 3; do - if nix develop --impure --command \ - bazel test //... \ - --verbose_failures \ - --lockfile_mode=error \ - 2>&1 | tee bazel.log; then - exit 0 - fi - grep -E '^ERROR:' bazel.log \ - | grep -Eq 'GET returned (403|429|5[0-9][0-9])|Bad Gateway|Connection (reset|timed out)|read timed out|Could not resolve host' || { - echo "::error::Bazel failed (non-transient); not retrying." - exit 1 - } - [ "$attempt" -lt 3 ] || break - echo "::warning::Transient fetch error (attempt $attempt); retrying in ${delay}s..." - sleep "$delay" - delay=$((delay * 2)) - done - echo "::error::Bazel still failing after 3 attempts." - exit 1 + nix develop --impure --command \ + bazel-retry test //... \ + --verbose_failures \ + --lockfile_mode=error \ - name: Teardown Worker uses: ./.github/actions/end-nix diff --git a/flake.nix b/flake.nix index dee3e0b77..0b71c013f 100644 --- a/flake.nix +++ b/flake.nix @@ -515,6 +515,30 @@ unset TMPDIR TMP exec ${pkgs.bazelisk}/bin/bazelisk "$@" ''; + bazel-retry = pkgs.writeShellScriptBin "bazel-retry" '' + set -o pipefail + BAZEL_LOG=$(mktemp -t) + unset TMPDIR TMP + # Bazel's downloader retry only covers truncated downloads, not HTTP 5xx/403 failures, + # so a flaky fetch aborts the build with no retry. We retry the download in this case. + delay=5 + for attempt in 1 2 3; do + if exec ${pkgs.bazelisk}/bin/bazelisk "$@" | tee ''${BAZEL_LOG}; then + exit 0 + fi + grep -E '^ERROR:' ''${BAZEL_LOG} \ + | grep -Eq 'GET returned (403|429|5[0-9][0-9])|Bad Gateway|Connection (reset|timed out)|read timed out|Could not resolve host' || { + echo "Bazel failed (non-transient); not retrying" + exit 1 + } + [ "$attempt" -lt 3 ] || break + echo "Transient fetch error (attempt ''${attempt}); retrying in ''${delay}s..." + sleep "$delay" + delay=$((delay * 2)) + done + echo "Bazel still failing after 3 attempts." + exit 1 + ''; in [ # Development tooling @@ -527,6 +551,7 @@ # Rust bazel + bazel-retry pkgs.lre.stable-rust pkgs.lre.lre-rs.lre-rs-configs-gen pkgs.rust-analyzer diff --git a/templates/bazel/.github/workflows/lre.yaml b/templates/bazel/.github/workflows/lre.yaml index ef5e4eca2..7a81115c7 100644 --- a/templates/bazel/.github/workflows/lre.yaml +++ b/templates/bazel/.github/workflows/lre.yaml @@ -34,4 +34,4 @@ jobs: source-tag: v3.13.0 - name: Build project - run: nix develop --command bazel build ... + run: nix develop --command bazel-retry build ... From d16a1d70b438f8616c93de96857e2f9a29421d5f Mon Sep 17 00:00:00 2001 From: Alec Maliwanag Date: Tue, 28 Jul 2026 06:35:14 +0100 Subject: [PATCH 60/84] web: remove NativeLink Cloud product listings and links (#2637) Removes the Cloud pricing tier, comparison column, dev.nativelink.com signup links, and remaining NativeLink Cloud mentions from the marketing site and docs. Enterprise remains the managed option; internal CI usage and historical changelogs are untouched. Co-authored-by: Marcus Eagan --- .../docs/content/docs/deployment/metrics.mdx | 4 +- .../docs/deployment/persistent-workers.mdx | 2 +- .../docs/content/docs/explanations/lre.mdx | 5 +- .../content/docs/getting-started/on-prem.mdx | 20 ++-- web/apps/docs/content/docs/index.mdx | 4 +- .../docs/rbe/local-remote-execution.mdx | 7 +- .../docs/content/docs/rbe/nix-templates.mdx | 4 +- web/apps/web/app/layout.tsx | 2 +- web/apps/web/app/license/page.tsx | 4 +- web/apps/web/app/pricing/page.tsx | 99 +++++-------------- web/apps/web/app/product/page.tsx | 8 +- web/apps/web/app/terms/page.tsx | 2 +- web/apps/web/components/mcp-demo.tsx | 6 +- .../web/content/posts/Finetune_LLM_On_CPU.mdx | 4 +- 14 files changed, 59 insertions(+), 112 deletions(-) diff --git a/web/apps/docs/content/docs/deployment/metrics.mdx b/web/apps/docs/content/docs/deployment/metrics.mdx index dccd8c47f..8e8b3bdc4 100644 --- a/web/apps/docs/content/docs/deployment/metrics.mdx +++ b/web/apps/docs/content/docs/deployment/metrics.mdx @@ -11,7 +11,7 @@ metrics are available when the store is explicitly wrapped with the opt-in NativeLink metrics are licensed under the Business Source License. Individual developer cache use does not need a commercial license. Teams using metrics in shared, production, or commercial settings -can use NativeLink Cloud, Enterprise, or an intentionally +can use NativeLink Enterprise or an intentionally very inexpensive separate license. See the [license page](https://nativelink.com/license). @@ -292,7 +292,7 @@ storage: Metrics are one of the few Business Source License modules. Individual developer use is fine; shared, production, or commercial - settings need NativeLink Cloud, Enterprise, or an intentionally + settings need NativeLink Enterprise or an intentionally inexpensive commercial license — see the [license page](https://nativelink.com/license). diff --git a/web/apps/docs/content/docs/deployment/persistent-workers.mdx b/web/apps/docs/content/docs/deployment/persistent-workers.mdx index 61ce78d0e..a77c84043 100644 --- a/web/apps/docs/content/docs/deployment/persistent-workers.mdx +++ b/web/apps/docs/content/docs/deployment/persistent-workers.mdx @@ -15,7 +15,7 @@ cost is paid once; subsequent actions reuse the same process. Remote persistent workers are licensed under the Business Source License. Individual developer cache use does not need a commercial license. Teams using persistent workers in shared, production, or -commercial settings can use NativeLink Cloud, Enterprise, or an +commercial settings can use NativeLink Enterprise or an intentionally very inexpensive separate license. See the [license page](https://nativelink.com/license). diff --git a/web/apps/docs/content/docs/explanations/lre.mdx b/web/apps/docs/content/docs/explanations/lre.mdx index cac03a257..23f7b5761 100644 --- a/web/apps/docs/content/docs/explanations/lre.mdx +++ b/web/apps/docs/content/docs/explanations/lre.mdx @@ -92,9 +92,8 @@ The recommended flow: build --remote_executor=grpcs://TODO ``` - Fill these in with either your [dev.nativelink.com](https://dev.nativelink.com) - credentials or a self-hosted cluster with a worker capable of the - platform the example needs. A plain + Fill these in with a self-hosted cluster that has a worker capable + of the platform the example needs. A plain [local instance](/rbe/local-testing) (drop the `s` in `grpcs://`, point at `127.0.0.1`) is enough to validate `remote_cache`, but the C++ example's `lre-cc` platform still needs a real worker diff --git a/web/apps/docs/content/docs/getting-started/on-prem.mdx b/web/apps/docs/content/docs/getting-started/on-prem.mdx index c7c998218..c7834b1c1 100644 --- a/web/apps/docs/content/docs/getting-started/on-prem.mdx +++ b/web/apps/docs/content/docs/getting-started/on-prem.mdx @@ -13,18 +13,19 @@ serves your team without 3 AM pages. Pick on-prem when one of these is true: - **Data residency.** Build artifacts can carry source code, - toolchains, even credentials. If those have to stay in a specific - region (EU GDPR, US FedRAMP boundaries, air-gapped corp networks), - managed Cloud isn't an option yet. + toolchains, even credentials. Self-hosting keeps them in whatever + region or boundary you need (EU GDPR, US FedRAMP boundaries, + air-gapped corp networks). - **Specialised hardware.** GPU workers, ARM cross-compile fleets, - in-house silicon. Cloud supports the common cases; on-prem lets you - use anything that can run a Linux binary. + in-house silicon. On-prem lets you use anything that can run a + Linux binary. - **You already operate stateful services.** If your team owns Kubernetes, Postgres, S3-compatible storage, adding NativeLink is marginal work. -If none of those apply, NativeLink Cloud is cheaper, faster to -provision, and one fewer pager rotation. +If you'd rather not run it yourself, +[Enterprise](https://nativelink.com/pricing) offers a managed +deployment. ## What ships in the box @@ -122,7 +123,7 @@ everything until it warms back up. for non-competing purposes are allowed. Metrics and remote persistent workers are Business Source License modules: fine for an individual developer's cache, but shared, production, or commercial use of those - modules needs NativeLink Cloud, Enterprise, or an intentionally + modules needs NativeLink Enterprise or an intentionally inexpensive commercial license. Meaningful contributors may be eligible for waivers — contact the maintainers before relying on one. The practical summary is on the @@ -132,7 +133,8 @@ everything until it warms back up. Compute for the control plane (a few small VMs), CAS storage — the dominant cost in a healthy cluster — worker compute, and the engineer hours to run it all. For most teams under ~30 engineers, self-hosting - is cheaper; past that, [Cloud](https://nativelink.com/pricing) + is cheaper; past that, a managed + [Enterprise](https://nativelink.com/pricing) deployment usually wins on engineer time alone. diff --git a/web/apps/docs/content/docs/index.mdx b/web/apps/docs/content/docs/index.mdx index a27d19973..5d64ef772 100644 --- a/web/apps/docs/content/docs/index.mdx +++ b/web/apps/docs/content/docs/index.mdx @@ -77,8 +77,8 @@ get most about that topic. The ones we get most overall: Yes — NativeLink is source-available and free for permitted self-hosted use. Most of the monorepo is licensed under `FSL-1.1-Apache-2.0`; a few modules (metrics, remote persistent - workers) are under the Business Source License, and paid - [Cloud and Enterprise](https://nativelink.com/pricing) tiers exist if + workers) are under the Business Source License, and a paid + [Enterprise](https://nativelink.com/pricing) tier exists if you'd rather not run it yourself. The licensing details are in [NativeLink on-prem](/getting-started/on-prem#faq). diff --git a/web/apps/docs/content/docs/rbe/local-remote-execution.mdx b/web/apps/docs/content/docs/rbe/local-remote-execution.mdx index 2a7afd755..49bd9e431 100644 --- a/web/apps/docs/content/docs/rbe/local-remote-execution.mdx +++ b/web/apps/docs/content/docs/rbe/local-remote-execution.mdx @@ -107,9 +107,8 @@ build --remote_timeout=600 build --remote_executor=grpcs://TODO ``` -Replace all three `TODO`s with either your -[dev.nativelink.com](https://dev.nativelink.com) credentials or your -own cluster's endpoint. Concretely, filled in for a plain local +Replace all three `TODO`s with your own cluster's endpoint. +Concretely, filled in for a plain local cluster (no TLS, so `grpc://` instead of `grpcs://`, and no BES endpoint to point at) — this is the exact `user.bazelrc` used to produce the verified output in step 4 above: @@ -120,7 +119,7 @@ build --remote_timeout=600 build --remote_executor=grpc://127.0.0.1:50051 ``` -A [dev.nativelink.com](https://dev.nativelink.com) or self-hosted +A TLS-terminated self-hosted cluster looks the same shape, just with `grpcs://` and your cluster's real hostnames in place of `127.0.0.1:50051`, plus a `bes_backend` line if you want build results streamed there. diff --git a/web/apps/docs/content/docs/rbe/nix-templates.mdx b/web/apps/docs/content/docs/rbe/nix-templates.mdx index 9c754f3ff..4d23818c4 100644 --- a/web/apps/docs/content/docs/rbe/nix-templates.mdx +++ b/web/apps/docs/content/docs/rbe/nix-templates.mdx @@ -53,13 +53,11 @@ build --remote_timeout=600 build --remote_executor=grpcs://TODO ``` -Three ways to fill in `TODO` — [Local Remote Execution → Point +Two ways to fill in `TODO` — [Local Remote Execution → Point `user.bazelrc` at a real cache and executor](/rbe/local-remote-execution) has a worked, filled-in example of the file below, whichever option you pick: -- **[dev.nativelink.com](https://dev.nativelink.com)** — paste your - cloud credentials, use `grpcs://`. - **A self-hosted cluster** — your own endpoint, `grpcs://` if it terminates TLS. - **Nothing but your own machine** — drop the `s`, point both diff --git a/web/apps/web/app/layout.tsx b/web/apps/web/app/layout.tsx index dadf1a346..90cc1dd27 100644 --- a/web/apps/web/app/layout.tsx +++ b/web/apps/web/app/layout.tsx @@ -37,7 +37,7 @@ export default function RootLayout({ children }: { children: ReactNode }) { Metrics and remote persistent workers are licensed under the Business Source License. Teams using those modules in shared, - production, or commercial settings should use NativeLink Cloud, - Enterprise, or a separate commercial license. + production, or commercial settings should use NativeLink + Enterprise or a separate commercial license.

    The separate commercial license is intentionally very inexpensive. diff --git a/web/apps/web/app/pricing/page.tsx b/web/apps/web/app/pricing/page.tsx index 762e0d80d..9552d1e5e 100644 --- a/web/apps/web/app/pricing/page.tsx +++ b/web/apps/web/app/pricing/page.tsx @@ -1,4 +1,4 @@ -import { Badge, Button, Eyebrow, Reveal, Section, cn } from "@nativelink/ui"; +import { Button, Eyebrow, Reveal, Section, cn } from "@nativelink/ui"; import { Fragment } from "react"; export const metadata = { title: "Pricing" }; @@ -17,24 +17,6 @@ const tiers = [ "All major cloud providers", ], cta: { label: "Get started", href: "/docs" }, - variant: "ghost" as const, - }, - { - name: "Cloud", - price: "$999+", - cadence: "/ month", - tagline: "Managed NativeLink. We run the infra, you ship.", - features: [ - "Fully managed cluster", - "Automatic scaling & failover", - "Production SLAs", - "Onboarding support", - "Dashboard & live build feed", - "Multi-region deployment", - ], - cta: { label: "Start free trial", href: "https://dev.nativelink.com/" }, - variant: "featured" as const, - badge: "Most popular", }, { name: "Enterprise", @@ -50,47 +32,45 @@ const tiers = [ "Priority feature requests", ], cta: { label: "Subscribe now", href: "https://enterprise.nativelink.com" }, - variant: "ghost" as const, }, ]; const comparison: { section: string; - rows: { label: string; oss: string | boolean; cloud: string | boolean; ent: string | boolean }[]; + rows: { label: string; oss: string | boolean; ent: string | boolean }[]; }[] = [ { section: "Platform", rows: [ - { label: "Hosting", oss: "Self-hosted", cloud: "Managed by us", ent: "On-prem or managed" }, - { label: "Distributed scheduler", oss: true, cloud: true, ent: true }, - { label: "Remote caching", oss: true, cloud: true, ent: true }, - { label: "Remote execution", oss: true, cloud: true, ent: true }, - { label: "Cross-compilation", oss: true, cloud: true, ent: true }, - { label: "External storage (S3, Redis)", oss: true, cloud: true, ent: true }, - { label: "Autoscaling", oss: false, cloud: true, ent: true }, - { label: "Multi-region", oss: "DIY", cloud: true, ent: true }, + { label: "Hosting", oss: "Self-hosted", ent: "On-prem or managed" }, + { label: "Distributed scheduler", oss: true, ent: true }, + { label: "Remote caching", oss: true, ent: true }, + { label: "Remote execution", oss: true, ent: true }, + { label: "Cross-compilation", oss: true, ent: true }, + { label: "External storage (S3, Redis)", oss: true, ent: true }, + { label: "Autoscaling", oss: false, ent: true }, + { label: "Multi-region", oss: "DIY", ent: true }, ], }, { section: "Compatibility", rows: [ - { label: "Supported build systems", oss: "All", cloud: "All", ent: "All + custom" }, + { label: "Supported build systems", oss: "All", ent: "All + custom" }, { label: "Operating systems", oss: "Linux, macOS", - cloud: "Linux, macOS, Windows", ent: "Linux, macOS, Windows", }, - { label: "Org-wide sharing", oss: true, cloud: true, ent: true }, + { label: "Org-wide sharing", oss: true, ent: true }, ], }, { section: "Operations", rows: [ - { label: "GUI dashboard", oss: false, cloud: true, ent: true }, - { label: "Build action breakdown", oss: false, cloud: true, ent: true }, - { label: "Live build updates", oss: false, cloud: true, ent: true }, - { label: "Audit logs / SSO", oss: false, cloud: true, ent: true }, + { label: "GUI dashboard", oss: false, ent: true }, + { label: "Build action breakdown", oss: false, ent: true }, + { label: "Live build updates", oss: false, ent: true }, + { label: "Audit logs / SSO", oss: false, ent: true }, ], }, { @@ -99,10 +79,9 @@ const comparison: { { label: "Channel", oss: "Community Slack", - cloud: "Email + Slack", ent: "Dedicated engineer", }, - { label: "Onboarding", oss: false, cloud: true, ent: "White-glove" }, + { label: "Onboarding", oss: false, ent: "White-glove" }, ], }, ]; @@ -142,8 +121,8 @@ export default function PricingPage() { .

    - Start free, self-host, or let us run your build farm. Three tiers that grow with - your team — no hidden fees, no per-user pricing. + Self-host for free, or let us run your build farm. Two tiers that grow with your + team — no hidden fees, no per-user pricing.

    @@ -152,24 +131,10 @@ export default function PricingPage() { {/* TIERS */}
    -
    +
    {tiers.map((tier, i) => ( -
    - {tier.variant === "featured" && tier.badge && ( -
    - - {tier.badge} - -
    - )} +

    {tier.name}

    @@ -187,12 +152,7 @@ export default function PricingPage() {
  • @@ -201,12 +161,7 @@ export default function PricingPage() { ))}
    -