diff --git a/.bazelrc b/.bazelrc index 6fdc8a0c1..91fb6415e 100644 --- a/.bazelrc +++ b/.bazelrc @@ -12,8 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -# Use blake3 as digest function instead of the much slower default sha256. -startup --digest_function=blake3 +# Keep Bazel aligned with NativeLink's default digest function. +startup --digest_function=sha256 common --enable_platform_specific_config # Don't use legacy toolchain resolution. @@ -158,7 +158,7 @@ build --@rules_rust//:clippy_flag=-Dclippy::use_debug build --@rules_rust//:clippy_flag=-Dclippy::used_underscore_binding build --@rules_rust//:clippy_flag=-Dclippy::useless_format build --@rules_rust//:clippy_flag=-Dclippy::cast_possible_truncation -build --@rules_rust//:clippy_flag=-Aclippy::cast_possible_wrap +build --@rules_rust//:clippy_flag=-Dclippy::cast_possible_wrap build --@rules_rust//:clippy_flag=-Aclippy::cast_precision_loss build --@rules_rust//:clippy_flag=-Aclippy::cast_sign_loss build --@rules_rust//:clippy_flag=-Aclippy::cognitive_complexity @@ -232,5 +232,8 @@ try-import %workspace%/nixos.bazelrc # Generated by the nativelink flake module. try-import %workspace%/nativelink.bazelrc +# Opt-in NativeLink Cloud configs used by this repo's CI. +try-import %workspace%/ci.bazelrc + # Allow user-side customization. try-import %workspace%/user.bazelrc diff --git a/.github/ISSUE_TEMPLATE/bug_report.yml b/.github/ISSUE_TEMPLATE/bug_report.yml new file mode 100644 index 000000000..d5ed0966a --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug_report.yml @@ -0,0 +1,120 @@ +name: Bug Report +description: File a bug report to help us improve NativeLink +title: "[Bug]: " +labels: ["type:bug"] +body: + - type: markdown + attributes: + value: | + Thanks for taking the time to fill out this bug report! Need help right away? + Join our [Slack community](https://forms.gle/LtaWSixEC6bYi5xF7) for faster responses. + + - type: input + id: version + attributes: + label: Version + description: What version of NativeLink are you running? + placeholder: "e.g. v1.0.0, main@abc1234, or ghcr.io image tag" + validations: + required: true + + - type: dropdown + id: client + attributes: + label: Client + description: Which client are you using with NativeLink? + options: + - Bazel + - Buck2 + - Other + validations: + required: true + + - type: input + id: os_arch + attributes: + label: OS / Architecture + description: What operating system and architecture? + placeholder: "e.g. Linux x86_64, macOS arm64" + validations: + required: true + + - type: dropdown + id: deployment + attributes: + label: Deployment method + description: How are you deploying NativeLink? + options: + - Docker + - Nix + - Kubernetes + - Bare metal + - Other + validations: + required: true + + - type: dropdown + id: storage + attributes: + label: Storage backend + description: Which storage backend are you using for the CAS? + options: + - S3 + - GCS + - Azure Blob + - Local filesystem + - Redis + - Other + validations: + required: true + + - type: input + id: workers + attributes: + label: Number of workers + description: How many workers are in your deployment? (if applicable) + placeholder: "e.g. 10" + + - type: input + id: cas_size + attributes: + label: Rough size of CAS + description: Approximately how large is your CAS store? + placeholder: "e.g. ~50GiB on local SSD" + + - type: textarea + id: config + attributes: + label: Configuration + description: Please paste a sanitized version of your NativeLink configuration. + render: json5 + placeholder: "{\n ...\n}" + + - type: textarea + id: expected_behavior + attributes: + label: Expected behavior + description: What did you expect to happen? + validations: + required: true + + - type: textarea + id: actual_behavior + attributes: + label: Actual behavior + description: What actually happened? + validations: + required: true + + - type: textarea + id: logs + attributes: + label: Logs / error output + description: Please paste any relevant logs, error messages, or stack traces. + render: shell + + - type: textarea + id: context + attributes: + label: Additional context + description: Anything else you think would be useful (links, references, screenshots, etc.). diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml new file mode 100644 index 000000000..1aedb2bc4 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -0,0 +1,8 @@ +blank_issues_enabled: true +contact_links: + - name: Join our Slack community + url: https://forms.gle/LtaWSixEC6bYi5xF7 + about: Get faster responses from the NativeLink community + - name: Commercial support + url: mailto:marcus@tracemachina.com + about: Contact us for commercial support inquiries diff --git a/.github/ISSUE_TEMPLATE/config_help.yml b/.github/ISSUE_TEMPLATE/config_help.yml new file mode 100644 index 000000000..773df8f98 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/config_help.yml @@ -0,0 +1,108 @@ +name: Configuration / Deployment Help +description: Ask for help with configuring, deploying, or troubleshooting NativeLink +title: "[Config/Help]: " +labels: ["type:question"] +body: + - type: markdown + attributes: + value: | + We're here to help! For faster responses, consider joining our + [Slack community](https://forms.gle/LtaWSixEC6bYi5xF7). For commercial + support, email us at contact@nativelink.com. + + - type: input + id: version + attributes: + label: Version + description: What version of NativeLink are you running? + placeholder: "e.g. v1.0.0, main@abc1234, or ghcr.io image tag" + validations: + required: true + + - type: dropdown + id: client + attributes: + label: Client + description: Which client are you using with NativeLink? + options: + - Bazel + - Buck2 + - Other + validations: + required: true + + - type: input + id: os_arch + attributes: + label: OS / Architecture + description: What operating system and architecture? + placeholder: "e.g. Linux x86_64, macOS arm64" + validations: + required: true + + - type: dropdown + id: deployment + attributes: + label: Deployment method + description: How are you deploying NativeLink? + options: + - Docker + - Nix + - Kubernetes + - Bare metal + - Other + validations: + required: true + + - type: dropdown + id: storage + attributes: + label: Storage backend + description: Which storage backend are you using for the CAS? + options: + - S3 + - GCS + - Azure Blob + - Local filesystem + - Redis + - Other + validations: + required: true + + - type: input + id: workers + attributes: + label: Number of workers + description: How many workers are in your deployment? (if applicable) + placeholder: "e.g. 10" + + - type: input + id: cas_size + attributes: + label: Rough size of CAS + description: Approximately how large is your CAS store? + placeholder: "e.g. ~50GiB on local SSD" + + - type: textarea + id: config + attributes: + label: Configuration + description: Please paste a sanitized version of your NativeLink configuration. + render: json5 + placeholder: "{\n ...\n}" + + - type: textarea + id: question + attributes: + label: What do you need help with? + description: Describe your question or issue in detail. + placeholder: "I'm trying to configure..." + validations: + required: true + + - type: textarea + id: tried + attributes: + label: What have you tried so far? + description: Steps you've already taken to resolve this. + placeholder: "I've tried..." diff --git a/.github/ISSUE_TEMPLATE/feature_request.yml b/.github/ISSUE_TEMPLATE/feature_request.yml new file mode 100644 index 000000000..4ab2a0edc --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature_request.yml @@ -0,0 +1,40 @@ +name: Feature Request +description: Suggest an idea or improvement for NativeLink +title: "[Feature]: " +labels: ["type:enhancement"] +body: + - type: markdown + attributes: + value: | + Thanks for sharing your idea! Need to discuss it first? + Join our [Slack community](https://forms.gle/LtaWSixEC6bYi5xF7) for faster feedback. + + - type: textarea + id: problem + attributes: + label: Problem statement + description: What problem are you trying to solve? What use case does this unlock? + placeholder: "I'm always frustrated when..." + validations: + required: true + + - type: textarea + id: solution + attributes: + label: Proposed solution + description: What would you like to see added or changed? + placeholder: "A new configuration option that..." + validations: + required: true + + - type: textarea + id: alternatives + attributes: + label: Alternatives considered + description: What workarounds or alternative solutions have you considered? + + - type: textarea + id: context + attributes: + label: Additional context + description: Any links, references, screenshots, or prior art that informed this request. diff --git a/.github/ISSUE_TEMPLATE/general.yml b/.github/ISSUE_TEMPLATE/general.yml new file mode 100644 index 000000000..e8a3b7086 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/general.yml @@ -0,0 +1,44 @@ +name: General Issue +description: For anything that doesn't fit the other categories +title: "[General]: " +labels: ["needs:triage"] +body: + - type: markdown + attributes: + value: | + Not sure if this is a bug or feature? Feel free to use this template. + For real-time help, join our [Slack community](https://forms.gle/LtaWSixEC6bYi5xF7). + + - type: dropdown + id: type + attributes: + label: Issue type + options: + - Question + - Discussion + - Suggestion + - Other + validations: + required: true + + - type: input + id: version + attributes: + label: Version (if applicable) + description: What version of NativeLink are you running? + placeholder: "e.g. v1.0.0, main@abc1234" + + - type: textarea + id: description + attributes: + label: Description + description: What's on your mind? + placeholder: "Tell us what you're thinking about..." + validations: + required: true + + - type: textarea + id: context + attributes: + label: Additional context + description: Any links, references, or screenshots that help explain. diff --git a/.github/actions/free-disk/action.yaml b/.github/actions/free-disk/action.yaml index 2bf00b955..77e405e4b 100644 --- a/.github/actions/free-disk/action.yaml +++ b/.github/actions/free-disk/action.yaml @@ -13,18 +13,26 @@ runs: endersonmenezes/free-disk-space@7901478139cff6e9d44df5972fd8ab8fcade4db1 with: rm_cmd: "rmz" # For speed up + rmz_version: "3.2.0" remove_android: false # Takes too long. remove_dotnet: true remove_haskell: true - remove_tool_cache: false # TODO(palfrey): Do we really need this? - # Note: Not deleting google-cloud-cli because it takes too long. + remove_tool_cache: false remove_folders: > + /opt/go + /opt/Ruby + /opt/microsoft/msedge + /usr/local/lib/android /usr/share/swift /usr/share/miniconda /usr/share/az* /usr/share/glade* /usr/local/share/chromium /usr/local/share/powershell + /usr/local/julia + /usr/local/aws-cli + /usr/local/aws-sam-cli + /usr/share/gradle # using hints from https://github.com/actions/runner-images/issues/10511#issuecomment-3984466720 - if: runner.os == 'macOS' diff --git a/.github/actions/prepare-nix/action.yaml b/.github/actions/prepare-nix/action.yaml index ef076e416..af22f0a35 100644 --- a/.github/actions/prepare-nix/action.yaml +++ b/.github/actions/prepare-nix/action.yaml @@ -11,6 +11,9 @@ runs: uses: ./.github/actions/free-disk - name: Install Nix + # Stay on the Determinate installer: the nix-quick-install switch (Nix + # 2.34.7) makes the nix2container multi-arch image publish fail with a + # layer digest mismatch. See the tagged image workflow. uses: >- # https://github.com/DeterminateSystems/nix-installer-action/releases/tag/v22 DeterminateSystems/nix-installer-action@ef8a148080ab6020fd15196c2084a2eea5ff2d25 with: diff --git a/.github/actions/setup-nativelink-cloud/action.yaml b/.github/actions/setup-nativelink-cloud/action.yaml new file mode 100644 index 000000000..17306a00c --- /dev/null +++ b/.github/actions/setup-nativelink-cloud/action.yaml @@ -0,0 +1,184 @@ +--- +name: Setup NativeLink Cloud +description: >- + Writes a user.bazelrc that opts this job into NativeLink Cloud remote + cache/BES (and optionally remote execution) via the configs in ci.bazelrc. + Trusted lanes pass secret-fed credentials; fork PRs, which receive neither + secrets nor repository variables, fall back to the committed public + read-only defaults below (cache reads only — no uploads, no BES, no remote + execution). ci-mode 'off' disables trusted lanes; the public path's kill + switch is revoking the public key server-side. + +inputs: + api-key: + description: >- + NativeLink Cloud API key. Empty (fork PRs have no secrets) disables the + setup entirely. + required: false + default: '' + claim-base: + description: >- + Claim endpoint base as ., normally passed from + secrets.NATIVELINK_STAGING_CLAIM_BASE. The action derives the + grpcs://cas-, bes-, and scheduler- endpoints from it. Empty disables + the setup. + required: false + default: '' + bes-results-url: + description: >- + Optional console URL prefix for invocation links, normally passed from + secrets.NATIVELINK_STAGING_BES_RESULTS_URL. Omitted from the build when + empty. + required: false + default: '' + public-read-key: + description: >- + World-readable cache_read-only API key used when api-key/claim-base are + unavailable (fork PRs receive neither secrets nor repository variables, + so the value must live here as a committed default to reach them). + Deliberately public: fork code runs with the value in hand either way, + so it must never carry more than cache_read. Kill switch is revoking + the key server-side; rotation is editing this default. + required: false + default: '66532088547d98bfb5b0b41e8d5cbfea8b5da1bc1e23e776f9cefc12654c6cb6' + public-claim-base: + description: >- + Claim endpoint base for the public read-only path. Committed default + for the same reason as public-read-key. + required: false + default: 'marcus-eagan-h6a62e.staging.scdev.nativelink.net' + ci-mode: + description: >- + Kill switch, normally passed as vars.NATIVELINK_CI_MODE. Empty or 'off' + disables; 'cache' enables remote cache + BES; 'rbe' additionally honors + exec: on. The canary workflow pins this to 'cache'/'rbe' so it stays + live while regular lanes are dark. + required: false + default: '' + mode: + description: >- + 'read' (PR lanes; never uploads action results) or 'write' (main-branch + push lanes; only TraceMachina main-branch identities hold write keys). + required: false + default: read + exec: + description: >- + 'on' requests remote execution. Honored only when ci-mode is 'rbe', and + only on Linux runners. + required: false + default: 'off' + container-image: + description: >- + Exec property attached when remote execution is honored. Must + byte-for-byte match the container-image the claim's worker pool + advertises, or actions queue forever. + required: false + default: debian:bookworm-slim + +outputs: + enabled: + description: "'true' when a user.bazelrc was written, else 'false'." + value: ${{ steps.configure.outputs.enabled }} + +runs: + using: composite + steps: + - name: Write user.bazelrc + id: configure + shell: bash + env: + NL_API_KEY: ${{ inputs.api-key }} + NL_CLAIM_BASE: ${{ inputs.claim-base }} + NL_BES_RESULTS_URL: ${{ inputs.bes-results-url }} + NL_PUBLIC_KEY: ${{ inputs.public-read-key }} + NL_PUBLIC_CLAIM_BASE: ${{ inputs.public-claim-base }} + NL_CI_MODE: ${{ inputs.ci-mode }} + NL_MODE: ${{ inputs.mode }} + NL_EXEC: ${{ inputs.exec }} + NL_CONTAINER_IMAGE: ${{ inputs.container-image }} + run: | + set -euo pipefail + + cred_kind="" + if [[ "$NL_CI_MODE" != "off" ]]; then + if [[ -n "$NL_API_KEY" && -n "$NL_CLAIM_BASE" ]]; then + # Private credentials also require the kill-switch variable to be + # explicitly on; never silently downgrade a trusted lane to the + # public path. + if [[ -n "$NL_CI_MODE" ]]; then + cred_kind="private" + fi + elif [[ -n "$NL_PUBLIC_KEY" && -n "$NL_PUBLIC_CLAIM_BASE" ]]; then + # Fork runs receive neither secrets nor repository variables, so + # the public read-only path cannot depend on ci-mode; revoking + # the public key server-side is its kill switch. + cred_kind="public" + fi + fi + + if [[ -z "$cred_kind" ]]; then + echo "enabled=false" >> "$GITHUB_OUTPUT" + echo "NativeLink Cloud disabled (no credentials or ci-mode off); building locally." + exit 0 + fi + + exec_enabled=false + if [[ "$NL_EXEC" == "on" ]]; then + if [[ "$RUNNER_OS" != "Linux" ]]; then + # macOS auto-applies LRE platforms from lre.bazelrc that conflict + # with the nl-rbe Linux platform, and no non-Linux workers exist. + echo "exec: on is only supported on Linux runners (got $RUNNER_OS)" >&2 + exit 1 + fi + # Remote execution needs a write-capable identity; the public + # read-only path never executes remotely. + if [[ "$NL_CI_MODE" == "rbe" && "$cred_kind" == "private" ]]; then + exec_enabled=true + fi + fi + + if [[ "$cred_kind" == "public" ]]; then + { + echo "common --config=nl-cache" + echo "common --remote_cache=grpcs://cas-${NL_PUBLIC_CLAIM_BASE}" + # The public key carries cache_read only: no uploads, no BES. + echo "common --remote_upload_local_results=false" + echo "common --remote_header=x-nativelink-api-key=${NL_PUBLIC_KEY}" + echo "common --remote_header=x-nativelink-project=nativelink-github-ci" + echo "common --build_metadata=ROLE=CI" + } >> user.bazelrc + echo "enabled=true" >> "$GITHUB_OUTPUT" + echo "NativeLink Cloud enabled: public read-only cache (no uploads, no BES)" + exit 0 + fi + + { + echo "common --config=nl-cache" + echo "common --remote_cache=grpcs://cas-${NL_CLAIM_BASE}" + echo "common --bes_backend=grpcs://bes-${NL_CLAIM_BASE}" + if [[ -n "$NL_BES_RESULTS_URL" ]]; then + echo "common --bes_results_url=${NL_BES_RESULTS_URL}" + fi + if [[ "$exec_enabled" == true ]]; then + echo "common --remote_executor=grpcs://scheduler-${NL_CLAIM_BASE}:443" + echo "common --remote_default_exec_properties=container-image=${NL_CONTAINER_IMAGE}" + echo "common --config=nl-rbe" + fi + if [[ "$NL_MODE" == "write" ]]; then + echo "common --config=nl-write" + else + # Plain (non-config) line so it also beats the nix-generated + # nativelink.bazelrc on lanes that run under nix develop. + echo "common --remote_upload_local_results=false" + fi + # Both headers from the same variable: a truncated BES key once + # caused silent 401s while the cache header kept working. + echo "common --remote_header=x-nativelink-api-key=${NL_API_KEY}" + echo "common --bes_header=x-nativelink-api-key=${NL_API_KEY}" + echo "common --remote_header=x-nativelink-project=nativelink-github-ci" + echo "common --build_metadata=ROLE=CI" + } >> user.bazelrc + + echo "enabled=true" >> "$GITHUB_OUTPUT" + # Deliberately do not echo the endpoints: the claim base is a secret. + echo "NativeLink Cloud enabled: mode=${NL_MODE} exec=${exec_enabled}" diff --git a/.github/actions/test-and-upload-image/action.yaml b/.github/actions/test-and-upload-image/action.yaml index c90ede9c9..21bff1073 100644 --- a/.github/actions/test-and-upload-image/action.yaml +++ b/.github/actions/test-and-upload-image/action.yaml @@ -23,6 +23,11 @@ inputs: runs: using: "composite" steps: + - name: Upgrade registries config to v2 + run: | + echo 'unqualified-search-registries = ["docker.io", "quay.io"]' | sudo tee /etc/containers/registries.conf + shell: bash -euo pipefail {0} + # FIXME: merge the multi-arch and single-arch paths - name: Test multi-arch image if: ${{ inputs.multi-arch == 'true' && inputs.testing == 'true' }} diff --git a/.github/styles/config/vocabularies/TraceMachina/accept.txt b/.github/styles/config/vocabularies/TraceMachina/accept.txt index 65d05bcbe..53e867eb0 100644 --- a/.github/styles/config/vocabularies/TraceMachina/accept.txt +++ b/.github/styles/config/vocabularies/TraceMachina/accept.txt @@ -9,6 +9,7 @@ Bazelisk [Kk]eyless Sigstore Cloudflare +[Cc]oalescers? Colab composable CPUs @@ -102,12 +103,11 @@ Config Grafana GitHub Deno -Http +HTTP IPs Noop ONTAP -Redis -redis +[Rr]edis shipstorm hermiticity performant @@ -172,7 +172,7 @@ impl [Rr]epos rules_cc rules_go -Tls +TLS Zlib backplanes deserialize @@ -296,3 +296,44 @@ shellexpand pluggable [Cc]allout hostnames +prefetch +protos +Serde +camelCase +SDK +varint +prost +unstorable +stdout +args +uncomment +hashmap +Buildbarn +deserialization +Dockerised +Merkle +subtree +hardlink +multiplicatively +SELinux +[Pp]assthrough +[Rr]ecompression +[Rr]ecompressions +[Dd]ecodable +libzstd +windowLog +[Ss]eekable +[Tt]ranscodes +[Ii]nterop +[Dd]esynchronization +max_recompression_size +EXDEV +[Oo]bservability +[Uu]nguessable +fsync +max_concurrent_staged_uploads +max_concurrent_identity_ops +max_inline_commit_size +stage_timeout_s +commit_timeout_s +compressed_upload_idle_timeout_s diff --git a/.github/workflows/config-reference.yaml b/.github/workflows/config-reference.yaml index bc8516e62..74eca2d5b 100644 --- a/.github/workflows/config-reference.yaml +++ b/.github/workflows/config-reference.yaml @@ -26,9 +26,13 @@ on: workflow_dispatch: inputs: tag: - description: 'Existing release tag to regenerate the config reference for (e.g. v1.6.1).' - required: true + description: 'Existing release tag to regenerate the config reference for (e.g. v1.6.1) or blank for all' + required: false # Blank to allow full regen type: string + # Weekly docs regeneration to catch any cases where we've made updates but haven't changed anything + schedule: + # Sunday 4:13am randomly picked + - cron: 13 4 * * SUN permissions: read-all @@ -52,10 +56,12 @@ jobs: set -euo pipefail if [[ "${{ github.event_name }}" == "workflow_dispatch" ]]; then TAG="${{ inputs.tag }}" + elif [[ "${{ github.event_name }}" == "schedule" ]]; then + TAG= else TAG="${{ github.ref_name }}" fi - if [[ ! "${TAG}" =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]]; then + if [[ "${TAG}" != "" && "${TAG}" != "main" && ( ! "${TAG}" =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ) ]]; then echo "::error::'${TAG}' is not a release tag (vMAJOR.MINOR.PATCH); refusing to regenerate." exit 1 fi @@ -103,8 +109,8 @@ jobs: fi BRANCH="docs/config-reference-${TAG}" - git config user.name "github-actions[bot]" - git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + git config user.name "config reference bot" + git config user.email "bot@tracemachina.com" git switch -c "${BRANCH}" git add web/apps/docs git commit -m "docs(config-reference): regenerate for NativeLink ${TAG}" diff --git a/.github/workflows/custom-image.yaml b/.github/workflows/custom-image.yaml index 3de1ce34a..73827b220 100644 --- a/.github/workflows/custom-image.yaml +++ b/.github/workflows/custom-image.yaml @@ -29,7 +29,7 @@ jobs: runs-on: ubuntu-24.04 permissions: issues: write - pull-requests: read + pull-requests: write outputs: should_build: ${{ steps.check.outputs.should_build }} pr_sha: ${{ steps.check.outputs.pr_sha }} @@ -46,67 +46,78 @@ jobs: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd with: script: | - var fs = require('fs'); - var matrix = JSON.parse(fs.readFileSync('.github/images-matrix.json', 'utf8')); - var validImages = []; - for (const option of matrix['include']) { - validImages.push(option['image']); - } + const fs = require('fs'); + const matrix = JSON.parse(fs.readFileSync('.github/images-matrix.json', 'utf8')); + const validImages = matrix['include'].map((o) => o['image']); + + // Never fail the job over a courtesy comment (e.g. a read-only + // token); the build is what matters. + const tryComment = async (body) => { + try { + await github.rest.issues.createComment({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: context.payload.issue.number, + body, + }); + } catch (e) { + core.warning(`Could not post comment: ${e.message}`); + } + }; + + let image; + let prSha; + if (context.eventName === 'workflow_dispatch') { - core.setOutput('should_build', 'true'); - core.setOutput('pr_sha', context.sha); image = '${{ inputs.image }}'; - } - else if (context.eventName === 'issue_comment') { + prSha = context.sha; + } else if (context.eventName === 'issue_comment') { const body = context.payload.comment.body.trim(); const isPR = !!context.payload.issue.pull_request; - // Match /build-image or /build-image const match = body.match(/^\/build-image(?:\s+(\S+))?/i); - - if (isPR && match) { - const { data: pr } = await github.rest.pulls.get({ - owner: context.repo.owner, - repo: context.repo.repo, - pull_number: context.payload.issue.number - }); - - const image = match[1] || 'image'; - - if (!validImages.includes(image)) { - await github.rest.issues.createComment({ - owner: context.repo.owner, - repo: context.repo.repo, - issue_number: context.payload.issue.number, - body: `Unknown image: \`${image}\`\n\nValid options: ${validImages.map(i => `\`${i}\``).join(', ')}` - }); - core.setOutput('should_build', 'false'); - return; - } - - core.setOutput('should_build', 'true'); - core.setOutput('pr_sha', pr.head.sha); - core.setOutput('image', image); - + if (!isPR || !match) { + core.setOutput('should_build', 'false'); + return; + } + image = match[1]; + if (!image || !validImages.includes(image)) { + const hint = image + ? `Unknown image: \`${image}\`` + : 'Specify an image to build.'; + await tryComment( + `${hint}\n\nValid options: ${validImages.map((i) => `\`${i}\``).join(', ')}` + ); + core.setOutput('should_build', 'false'); + return; + } + const { data: pr } = await github.rest.pulls.get({ + owner: context.repo.owner, + repo: context.repo.repo, + pull_number: context.payload.issue.number, + }); + prSha = pr.head.sha; + try { await github.rest.reactions.createForIssueComment({ owner: context.repo.owner, repo: context.repo.repo, comment_id: context.payload.comment.id, - content: 'rocket' + content: 'rocket', }); + } catch (e) { + core.warning(`Could not add reaction: ${e.message}`); } } else { core.setOutput('should_build', 'false'); - return + return; } + + const option = matrix['include'].find((o) => o['image'] === image); + core.setOutput('should_build', 'true'); + core.setOutput('pr_sha', prSha); core.setOutput('image', image); - for (const option of matrix['include']) { - if (option['image'] == image) { - core.setOutput('multi-arch', option['multi-arch'] ?? false); - core.setOutput('components', option['components'] ?? ""); - return - } - } + core.setOutput('multi-arch', option?.['multi-arch'] ?? false); + core.setOutput('components', option?.['components'] ?? ''); build-image: name: Build and Push Image @@ -116,6 +127,7 @@ jobs: permissions: contents: read issues: write + pull-requests: write timeout-minutes: 45 steps: - name: Checkout @@ -155,14 +167,20 @@ jobs: if: github.event_name == 'issue_comment' uses: >- #v8.0.0 actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd + env: + IMAGE_REF: ghcr.io/${{ github.repository_owner }}/${{ needs.check-trigger.outputs.image }}:${{ steps.version.outputs.version-string }} with: script: | - await github.rest.issues.createComment({ - owner: context.repo.owner, - repo: context.repo.repo, - issue_number: context.payload.issue.number, - body: `Image built and pushed!\n\n\`\`\`\n${{ steps.upload.outputs.image_tag }}\n\`\`\`` - }); + try { + await github.rest.issues.createComment({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: context.payload.issue.number, + body: `Image built and pushed!\n\n\`\`\`\n${process.env.IMAGE_REF}\n\`\`\``, + }); + } catch (e) { + core.warning(`Could not post comment: ${e.message}`); + } - name: Teardown Worker uses: ./.github/actions/end-nix diff --git a/.github/workflows/image.yaml b/.github/workflows/image.yaml index 37a773e55..e5f473891 100644 --- a/.github/workflows/image.yaml +++ b/.github/workflows/image.yaml @@ -51,13 +51,6 @@ jobs: uses: >- # v6.0.2 actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd - # Skopeo login otherwise complains with - # "get credentials: loading registries configuration \"/etc/containers/registries.conf\": registries.conf must be in v2 format but is in v1" - - name: Upgrade registries config to v2 - run: | - cat /etc/containers/registries.conf - echo "unqualified-search-registries = [\"docker.io\", \"quay.io\"]" | sudo tee /etc/containers/registries.conf - - name: Prepare Worker uses: ./.github/actions/prepare-nix with: diff --git a/.github/workflows/lre.yaml b/.github/workflows/lre.yaml index 61927eca8..9831e6ed9 100644 --- a/.github/workflows/lre.yaml +++ b/.github/workflows/lre.yaml @@ -42,12 +42,24 @@ jobs: with: nativelink_attic_token: ${{ secrets.NATIVELINK_ATTIC_TOKEN }} + - name: Setup NativeLink Cloud + uses: ./.github/actions/setup-nativelink-cloud + with: + ci-mode: ${{ vars.NATIVELINK_CI_MODE }} + claim-base: ${{ secrets.NATIVELINK_STAGING_CLAIM_BASE }} + bes-results-url: ${{ secrets.NATIVELINK_STAGING_BES_RESULTS_URL }} + api-key: >- + ${{ github.event_name == 'push' + && secrets.NATIVELINK_STAGING_API_KEY_RW + || secrets.NATIVELINK_STAGING_API_KEY_RO }} + mode: ${{ github.event_name == 'push' && 'write' || 'read' }} + - name: Build example with ${{ matrix.toolchain }} toolchain env: TOOLCHAIN: ${{ matrix.toolchain }} run: > nix develop --impure --command - bash -c "bazel run \ + bash -c "bazel-retry run \ --lockfile_mode=error \ --verbose_failures \ @local-remote-execution//examples:${TOOLCHAIN}" diff --git a/.github/workflows/native-bazel.yaml b/.github/workflows/native-bazel.yaml index b38d65a74..7161dab46 100644 --- a/.github/workflows/native-bazel.yaml +++ b/.github/workflows/native-bazel.yaml @@ -60,6 +60,18 @@ jobs: repository-cache: true disk-cache: ${{ github.workflow }}-${{ matrix.os }}-bazel-${{ matrix.bazel_version }} + - name: Setup NativeLink Cloud + uses: ./.github/actions/setup-nativelink-cloud + with: + ci-mode: ${{ vars.NATIVELINK_CI_MODE }} + claim-base: ${{ secrets.NATIVELINK_STAGING_CLAIM_BASE }} + bes-results-url: ${{ secrets.NATIVELINK_STAGING_BES_RESULTS_URL }} + api-key: >- + ${{ github.event_name == 'push' + && secrets.NATIVELINK_STAGING_API_KEY_RW + || secrets.NATIVELINK_STAGING_API_KEY_RO }} + mode: ${{ github.event_name == 'push' && 'write' || 'read' }} + - name: Show Bazel version run: bazel --version @@ -115,6 +127,18 @@ jobs: repository-cache: true disk-cache: ${{ github.workflow }}-ubuntu-24.04 + - name: Setup NativeLink Cloud + uses: ./.github/actions/setup-nativelink-cloud + with: + ci-mode: ${{ vars.NATIVELINK_CI_MODE }} + claim-base: ${{ secrets.NATIVELINK_STAGING_CLAIM_BASE }} + bes-results-url: ${{ secrets.NATIVELINK_STAGING_BES_RESULTS_URL }} + api-key: >- + ${{ github.event_name == 'push' + && secrets.NATIVELINK_STAGING_API_KEY_RW + || secrets.NATIVELINK_STAGING_API_KEY_RO }} + mode: ${{ github.event_name == 'push' && 'write' || 'read' }} + - name: Run Store tester with sentinel run: | bazel run //:redis_store_tester \ diff --git a/.github/workflows/native-cargo.yaml b/.github/workflows/native-cargo.yaml index 2b6789598..1bc1a4523 100644 --- a/.github/workflows/native-cargo.yaml +++ b/.github/workflows/native-cargo.yaml @@ -47,8 +47,8 @@ jobs: run: rustup update && rustup default ${{ matrix.toolchain }} - name: Rust cache - # https://github.com/Swatinem/rust-cache/releases/tag/v2.9.1 - uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 + uses: >- # v2.9.1 + Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 - name: Build on ${{ runner.os }} run: cargo build --all --profile=smol diff --git a/.github/workflows/nativelink-cloud-canary.yaml b/.github/workflows/nativelink-cloud-canary.yaml new file mode 100644 index 000000000..2539ac7de --- /dev/null +++ b/.github/workflows/nativelink-cloud-canary.yaml @@ -0,0 +1,106 @@ +--- +name: NativeLink Cloud Canary + +on: + schedule: + - cron: '17 6 * * *' + workflow_dispatch: + pull_request: + branches: [main] + paths: + - ci.bazelrc + - .github/actions/setup-nativelink-cloud/** + - .github/workflows/nativelink-cloud-canary.yaml + +permissions: read-all + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} + +jobs: + cache-canary: + name: Remote cache canary + runs-on: ubuntu-24.04 + # Also the queue-forever tripwire once an RBE leg is added. + timeout-minutes: 15 + steps: + - name: Checkout + uses: >- # v6.0.2 + actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd + + - name: Setup Bazel + uses: >- # v0.19.0 + bazel-contrib/setup-bazel@c5acdfb288317d0b5c0bbd7a396a3dc868bb0f86 + with: + bazelisk-cache: true + repository-cache: true + # No disk-cache: remote hits must be the only cache signal here. + + - name: Setup NativeLink Cloud + id: nl + uses: ./.github/actions/setup-nativelink-cloud + with: + # The canary ignores vars.NATIVELINK_CI_MODE on purpose: it must + # prove the cloud path is healthy while regular lanes are still + # dark. With no secrets configured (or on fork PRs) it no-ops. + ci-mode: cache + claim-base: ${{ secrets.NATIVELINK_STAGING_CLAIM_BASE }} + bes-results-url: ${{ secrets.NATIVELINK_STAGING_BES_RESULTS_URL }} + api-key: >- + ${{ github.event_name == 'pull_request' + && secrets.NATIVELINK_STAGING_API_KEY_RO + || secrets.NATIVELINK_STAGING_API_KEY_RW }} + mode: ${{ github.event_name == 'pull_request' && 'read' || 'write' }} + + - name: Build (populate cache) + if: steps.nl.outputs.enabled == 'true' + run: | + bazel build //nativelink-config/... \ + --lockfile_mode=error \ + --verbose_failures \ + --bes_upload_mode=wait_for_upload_complete + + # The hit-rate assertion needs the populate step to have uploaded, so it + # only runs with the write key (schedule/dispatch/push — never PRs). + - name: Drop local build state + if: steps.nl.outputs.enabled == 'true' && github.event_name != 'pull_request' + run: | + bazel clean + bazel shutdown + + - name: Rebuild and assert remote cache hits + if: steps.nl.outputs.enabled == 'true' && github.event_name != 'pull_request' + run: | + bazel build //nativelink-config/... \ + --lockfile_mode=error \ + --verbose_failures \ + --bes_upload_mode=wait_for_upload_complete \ + --build_event_json_file="$RUNNER_TEMP/bep.json" + + hits=$(jq -s '[.[] | .buildMetrics.actionSummary.runnerCount[]? + | select(.name == "remote cache hit") | .count] | add // 0' \ + "$RUNNER_TEMP/bep.json") + # runnerCount includes an aggregate "total" entry alongside the + # per-runner entries; count only real runners or the rate is halved. + total=$(jq -s '[.[] | .buildMetrics.actionSummary.runnerCount[]? + | select(.name != "internal" and .name != "total") | .count] | add // 0' \ + "$RUNNER_TEMP/bep.json") + invocation=$(jq -rs '[.[] | select(.id.started != null) + | .started.uuid][0] // empty' "$RUNNER_TEMP/bep.json") + + echo "Remote cache hits on rebuild: ${hits}/${total} (invocation ${invocation})" + { + echo "### NativeLink Cloud canary" + echo "- Remote cache hits on rebuild: **${hits}/${total}**" + echo "- Invocation: \`${invocation}\`" + } >> "$GITHUB_STEP_SUMMARY" + + if [[ "$total" -eq 0 ]]; then + echo "FAIL: no actions were measured; the assertion is vacuous" >&2 + exit 1 + fi + if (( hits * 100 < total * 90 )); then + echo "FAIL: remote cache hit rate below 90% on an unchanged rebuild" >&2 + exit 1 + fi diff --git a/.github/workflows/nix.yaml b/.github/workflows/nix.yaml index 009061aec..378b9bff4 100644 --- a/.github/workflows/nix.yaml +++ b/.github/workflows/nix.yaml @@ -38,33 +38,24 @@ jobs: with: nativelink_attic_token: ${{ secrets.NATIVELINK_ATTIC_TOKEN }} + - name: Setup NativeLink Cloud + uses: ./.github/actions/setup-nativelink-cloud + with: + ci-mode: ${{ vars.NATIVELINK_CI_MODE }} + claim-base: ${{ secrets.NATIVELINK_STAGING_CLAIM_BASE }} + bes-results-url: ${{ secrets.NATIVELINK_STAGING_BES_RESULTS_URL }} + api-key: >- + ${{ github.event_name == 'push' + && secrets.NATIVELINK_STAGING_API_KEY_RW + || secrets.NATIVELINK_STAGING_API_KEY_RO }} + mode: ${{ github.event_name == 'push' && 'write' || 'read' }} + - name: Invoke Bazel build in Nix shell run: | - set -o pipefail - - # Bazel's downloader retry only covers truncated downloads, not HTTP 5xx/403 failures, - # so a flaky GitHub fetch aborts the build with no retry. We retry the download in this case. - delay=5 - for attempt in 1 2 3; do - if nix develop --impure --command \ - bazel test //... \ - --verbose_failures \ - --lockfile_mode=error \ - 2>&1 | tee bazel.log; then - exit 0 - fi - grep -E '^ERROR:' bazel.log \ - | grep -Eq 'GET returned (403|429|5[0-9][0-9])|Bad Gateway|Connection (reset|timed out)|read timed out|Could not resolve host' || { - echo "::error::Bazel failed (non-transient); not retrying." - exit 1 - } - [ "$attempt" -lt 3 ] || break - echo "::warning::Transient fetch error (attempt $attempt); retrying in ${delay}s..." - sleep "$delay" - delay=$((delay * 2)) - done - echo "::error::Bazel still failing after 3 attempts." - exit 1 + nix develop --impure --command \ + bazel-retry test //... \ + --verbose_failures \ + --lockfile_mode=error \ - name: Teardown Worker uses: ./.github/actions/end-nix @@ -90,6 +81,10 @@ jobs: with: nativelink_attic_token: ${{ secrets.NATIVELINK_ATTIC_TOKEN }} + - name: Rust cache + uses: >- # v2.9.1 + Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 + - name: Invoke Cargo build in Nix shell run: > nix develop --impure --command diff --git a/.github/workflows/sanitizers.yaml b/.github/workflows/sanitizers.yaml index 0a867f368..2eb7aa65c 100644 --- a/.github/workflows/sanitizers.yaml +++ b/.github/workflows/sanitizers.yaml @@ -44,5 +44,17 @@ jobs: bazelisk-cache: true repository-cache: true + - name: Setup NativeLink Cloud + uses: ./.github/actions/setup-nativelink-cloud + with: + ci-mode: ${{ vars.NATIVELINK_CI_MODE }} + claim-base: ${{ secrets.NATIVELINK_STAGING_CLAIM_BASE }} + bes-results-url: ${{ secrets.NATIVELINK_STAGING_BES_RESULTS_URL }} + api-key: >- + ${{ github.event_name == 'push' + && secrets.NATIVELINK_STAGING_API_KEY_RW + || secrets.NATIVELINK_STAGING_API_KEY_RO }} + mode: ${{ github.event_name == 'push' && 'write' || 'read' }} + - name: Run Bazel tests run: bazel test --config=${{ matrix.sanitizer }} --lockfile_mode=error --verbose_failures //... diff --git a/.vale.ini b/.vale.ini index 07da1def9..376e5cc25 100644 --- a/.vale.ini +++ b/.vale.ini @@ -12,6 +12,51 @@ IgnoredScopes = code, tt, frontmatter [formats] mdx = md +rs = md + +# FIXME(palfrey): really this should be all our rust code, but that's a lot of changes +# Focusing first on the config bits that get generated into our reference code +# When we do that, we can de-dupe this section and the markdown-specific one +[nativelink-config/src/*.rs] +BasedOnStyles = alex, Vale, Microsoft, write-good + +# Ignore code blocks in Starlight's TabItems. +BlockIgnores = (?s)(.*?```.*?```.*?) + +# Skip URL targets in Markdown links — Vale.Terms shouldn't lint URLs, +# but does by default. The `(?<=\])` lookbehind narrows the pattern so +# it only matches the URL portion of `[link text](url)`. +TokenIgnores = ['"]\.\/[\w-]+\.mdx['"], (?<=\])\([^)]+\) + +# Too harsh. The `write-good.Passive` check already covers many cases. +write-good.E-Prime = NO + +# Redundant. Covered by `write-good.Passive`. +Microsoft.Passive = NO + +# We use typographic em-dashes with spaces for readability. The Microsoft +# style guide wants tight em-dashes; we don't. +Microsoft.Dashes = NO + +# Pedantic style rules we don't want to enforce in technical docs: +Microsoft.Avoid = NO # blanket bans on words like "backend" +Microsoft.HeadingColons = NO # nitpicks heading punctuation +Microsoft.Contractions = NO # "what's" vs "what is" is a tone choice +Microsoft.Foreign = NO # allows "e.g." and "i.e." +Microsoft.Quotes = NO # punctuation-inside-quotes is style +Microsoft.We = NO # we use "we" in conversational tech docs +Microsoft.Negative = NO # en-dash isn't a good idea for negatives + +# Vale's built-in Terms rule does case-sensitive substitution. In tech +# docs, the same word can be a lowercase command (`bazel build`), a +# capitalised proper noun (Bazel), or a path fragment in a URL. Trying +# to enforce one form across all three is noisy and produces false +# positives in code blocks and URLs. +Vale.Terms = NO + +# Microsoft.Auto wants "autodetects" instead of "auto-detects". Both are +# fine in technical prose. +Microsoft.Auto = NO [*.{md,mdx}] BasedOnStyles = alex, Vale, Microsoft, write-good diff --git a/CHANGELOG.md b/CHANGELOG.md index 0588c1eda..02cb69222 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,101 @@ All notable changes to this project will be documented in this file. +## [1.6.3](https://github.com/TraceMachina/nativelink/compare/v1.6.2..v1.6.3) - 2026-07-24 + +### ⛰️ Features + +- Add Google Tag Manager to the marketing site ([#2610](https://github.com/TraceMachina/nativelink/issues/2610)) - ([94f312a](https://github.com/TraceMachina/nativelink/commit/94f312a043847d5dd0f8ca95b676b00d4697c04d)) +- Add the Leadfeeder tracker to the marketing site ([#2606](https://github.com/TraceMachina/nativelink/issues/2606)) - ([9700a4a](https://github.com/TraceMachina/nativelink/commit/9700a4a522de525b9121188f19bd1c74b2d4a10e)) +- Prefetch directory-cache tree protos with one GetTree stream ([#2546](https://github.com/TraceMachina/nativelink/issues/2546)) - ([2cbf21e](https://github.com/TraceMachina/nativelink/commit/2cbf21ef52f78ea95d0eb11f9f4396022619b70d)) + +### 🐛 Bug Fixes + +- Fix #2564, .exec cleanup and coun F_FULLFSYNC operations ([#2602](https://github.com/TraceMachina/nativelink/issues/2602)) - ([b44f31d](https://github.com/TraceMachina/nativelink/commit/b44f31dde49cd89210a9d10fdd5414b56db741d1)) +- Fix custom-image workflow: grant pull-requests: write and fix command parsing ([#2600](https://github.com/TraceMachina/nativelink/issues/2600)) - ([63567c8](https://github.com/TraceMachina/nativelink/commit/63567c85b8fc609e47b990336c55b91133069ac1)) +- Let GCS get_part defer NotFound to the retrier ([#2584](https://github.com/TraceMachina/nativelink/issues/2584)) - ([3947c71](https://github.com/TraceMachina/nativelink/commit/3947c71603b5d3acb39d2f0caf2a22db6b009089)) +- Fix skopeo login on ubuntu-24.04 for all image workflows ([#2577](https://github.com/TraceMachina/nativelink/issues/2577)) - ([9717f2f](https://github.com/TraceMachina/nativelink/commit/9717f2fb432c0b8cdd26b078d4851040be64abdd)) + +### 📚 Documentation + +- *(config-reference)* regenerate for NativeLink v1.6.2 ([#2576](https://github.com/TraceMachina/nativelink/issues/2576)) - ([9daa989](https://github.com/TraceMachina/nativelink/commit/9daa989b24330a09f570c623d13331fd293e5062)) +- Move the FAQ into per-page FAQ sections ([#2590](https://github.com/TraceMachina/nativelink/issues/2590)) - ([82a036c](https://github.com/TraceMachina/nativelink/commit/82a036c4a211d6009a251fd47182686f0a2bd85f)) +- Extend the docs FAQ to cover every documentation area ([#2581](https://github.com/TraceMachina/nativelink/issues/2581)) - ([217276c](https://github.com/TraceMachina/nativelink/commit/217276cef05ba5cc2937676b0c874d223a184dac)) +- Redis eviction events support ([#2521](https://github.com/TraceMachina/nativelink/issues/2521)) - ([45a5652](https://github.com/TraceMachina/nativelink/commit/45a56520751e3266be7427bd61950a40eeb49ff0)) + +### 🧪 Testing & CI + +- Skip empty chunks in GCS get_part read stream ([#2620](https://github.com/TraceMachina/nativelink/issues/2620)) - ([41be332](https://github.com/TraceMachina/nativelink/commit/41be33237322c9a67a5431948945b2f7adde3524)) +- Add trace logging for bytestream read ([#2482](https://github.com/TraceMachina/nativelink/issues/2482)) - ([4e2f554](https://github.com/TraceMachina/nativelink/commit/4e2f554f76789edf1072c97db39b8f84906e6a3e)) +- Bootstrap CI on NativeLink Cloud staging (remote cache + BES, dark) ([#2579](https://github.com/TraceMachina/nativelink/issues/2579)) - ([eed5b31](https://github.com/TraceMachina/nativelink/commit/eed5b31e16d2853c2dffddfa107324cd0d7ba0a7)) + +### ⚙️ Miscellaneous + +- Web/compliance and careers ([#2624](https://github.com/TraceMachina/nativelink/issues/2624)) - ([97916b2](https://github.com/TraceMachina/nativelink/commit/97916b2af9b53f0042e7199ace4dfbc6b5c02b95)) +- Replace determinate with nix-quick-install-action ([#2618](https://github.com/TraceMachina/nativelink/issues/2618)) - ([bf908e5](https://github.com/TraceMachina/nativelink/commit/bf908e529eded52fca0a5006e414bb5e333d3be6)) +- add security and trust responsibilities to careers job description ([#2622](https://github.com/TraceMachina/nativelink/issues/2622)) - ([3d40c91](https://github.com/TraceMachina/nativelink/commit/3d40c91a103cc3fa1095f73c4dca13ca64f91f07)) +- Upgrade to rules_rs 0.0.96 ([#2615](https://github.com/TraceMachina/nativelink/issues/2615)) - ([0e7580f](https://github.com/TraceMachina/nativelink/commit/0e7580fcf10d8833e93938d7a76c524a9577fa4e)) +- Increase timeout on trivy ([#2613](https://github.com/TraceMachina/nativelink/issues/2613)) - ([fbf49d6](https://github.com/TraceMachina/nativelink/commit/fbf49d62a660b9bb6e33a7ceb1f4ddb203bd4216)) +- Record the execution metrics that were declared but never emitted ([#2599](https://github.com/TraceMachina/nativelink/issues/2599)) - ([7bb62d7](https://github.com/TraceMachina/nativelink/commit/7bb62d7f2f5b3c0c980b1939c98061204afaca51)) +- Make filesystem store posix_fadvise(DONTNEED) opt-in ([#2608](https://github.com/TraceMachina/nativelink/issues/2608)) - ([d15b2a9](https://github.com/TraceMachina/nativelink/commit/d15b2a97a2d599da520ac12fd45e832dd4f99147)) +- Commit public read-only cache defaults so fork PRs actually receive them ([#2604](https://github.com/TraceMachina/nativelink/issues/2604)) - ([9fe34de](https://github.com/TraceMachina/nativelink/commit/9fe34de2eb6a28b3414fa2e2016c56e657bda279)) +- Skip pre_exec hook when namespaces are disabled to allow posix_spawn ([#2597](https://github.com/TraceMachina/nativelink/issues/2597)) - ([02e3e76](https://github.com/TraceMachina/nativelink/commit/02e3e7665ef566da7eabb10a8d7622d6f0b9dbf3)) +- Coalesce FilesystemStore durability flushes on macOS ([#2539](https://github.com/TraceMachina/nativelink/issues/2539)) - ([97b961a](https://github.com/TraceMachina/nativelink/commit/97b961aac277dd27e0472930a3b1b4733467b8f3)) +- cmake links; rust headers; sandboxing ([#2586](https://github.com/TraceMachina/nativelink/issues/2586)) - ([ddcb47e](https://github.com/TraceMachina/nativelink/commit/ddcb47e5e2c0baaa9537fe5c4be87499f44cde24)) +- Exclude the aggregate total runner entry from the canary hit-rate math ([#2589](https://github.com/TraceMachina/nativelink/issues/2589)) - ([3b1e179](https://github.com/TraceMachina/nativelink/commit/3b1e179b835b03633ee2b42d4ef4fd6627bb57dd)) +- Give fork PRs read-only remote cache via a public key variable ([#2587](https://github.com/TraceMachina/nativelink/issues/2587)) - ([43c0658](https://github.com/TraceMachina/nativelink/commit/43c0658a65e75c073dddf575c3ef6cb8e9afcfe9)) +- cargo-llvm-cov gets flagged, but doesn't have actual vulnerabilities ([#2585](https://github.com/TraceMachina/nativelink/issues/2585)) - ([d59e50f](https://github.com/TraceMachina/nativelink/commit/d59e50f29cff85de3b6194e0bce2a65911830ca4)) +- Cache cargo runs under nix ([#2570](https://github.com/TraceMachina/nativelink/issues/2570)) - ([5ee0afc](https://github.com/TraceMachina/nativelink/commit/5ee0afcfbff2a2393ea3a31b5298e0f38ead1448)) + +### ⬆️ Bumps & Version Updates + +- fix compliance claims, normalize contact emails, add careers pages ([#2621](https://github.com/TraceMachina/nativelink/issues/2621)) - ([b3659b7](https://github.com/TraceMachina/nativelink/commit/b3659b7ee82ad4ffb7bfba6561c3bd10b093e22b)) +- add change history to Terms & Privacy page ([#2619](https://github.com/TraceMachina/nativelink/issues/2619)) - ([c3ffa86](https://github.com/TraceMachina/nativelink/commit/c3ffa86c2ecefeff2cc1afdc2d8bb69230008743)) +- anyhow to 1.0.104 ([#2580](https://github.com/TraceMachina/nativelink/issues/2580)) - ([d1e49ac](https://github.com/TraceMachina/nativelink/commit/d1e49ac5f547006c3ecba9e6a94b16ab82897dce)) +- Update Rust crate syn to v3 ([#2578](https://github.com/TraceMachina/nativelink/issues/2578)) - ([abd0258](https://github.com/TraceMachina/nativelink/commit/abd025888d9b553380ac169f73f802005987749b)) + +## [1.6.2](https://github.com/TraceMachina/nativelink/compare/v1.6.1..v1.6.2) - 2026-07-17 + +### ⛰️ Features + +- Add fallback match interval to simple scheduler ([#2557](https://github.com/TraceMachina/nativelink/issues/2557)) - ([7a4279b](https://github.com/TraceMachina/nativelink/commit/7a4279b4e51cfb6064937b50f763e3c13733db0a)) +- Add opt-in small-blob read coalescing to GrpcStore ([#2540](https://github.com/TraceMachina/nativelink/issues/2540)) - ([4be35e6](https://github.com/TraceMachina/nativelink/commit/4be35e6dccb7fbc08d414ca029c592e9e7853cbf)) + +### 🐛 Bug Fixes + +- Remove quoting around NetApp ([#2556](https://github.com/TraceMachina/nativelink/issues/2556)) - ([65e2ec2](https://github.com/TraceMachina/nativelink/commit/65e2ec2004ca28e3d9273959c7a809f9b0dcb0e0)) +- Fix skopeo login issues on image publish ([#2549](https://github.com/TraceMachina/nativelink/issues/2549)) - ([e72490c](https://github.com/TraceMachina/nativelink/commit/e72490cc3e388a944f177ce47bca46ecbb027180)) +- Handle zero digests in CompressionStore ([#2548](https://github.com/TraceMachina/nativelink/issues/2548)) - ([b08995d](https://github.com/TraceMachina/nativelink/commit/b08995de0447d92fc5a47f71af1d83d4c4da142c)) + +### 📚 Documentation + +- *(config-reference)* regenerate for NativeLink v1.6.0 ([#2553](https://github.com/TraceMachina/nativelink/issues/2553)) - ([f871377](https://github.com/TraceMachina/nativelink/commit/f871377c09ed6fea7538d372e3a3cd2240c231e2)) +- Adds weekly and "everything" docs regen ([#2551](https://github.com/TraceMachina/nativelink/issues/2551)) - ([2a061ba](https://github.com/TraceMachina/nativelink/commit/2a061ba865d2509decc762f11fbdbcf2871d04f9)) +- Remove StoreSpec docs ([#2545](https://github.com/TraceMachina/nativelink/issues/2545)) - ([fb37e86](https://github.com/TraceMachina/nativelink/commit/fb37e86317de19e757ff4e276f2021f9ae0b79cc)) +- Add opt-in subtree-keyed caching to the worker DirectoryCache ([#2541](https://github.com/TraceMachina/nativelink/issues/2541)) - ([9f9451e](https://github.com/TraceMachina/nativelink/commit/9f9451e02cfc1adf0c7830fa4003ab328f894d56)) + +### 🧪 Testing & CI + +- Fix AsyncFixedBuffer with zstd compression, remove unused BufChannelReader adapter ([#2574](https://github.com/TraceMachina/nativelink/issues/2574)) - ([b01cb0d](https://github.com/TraceMachina/nativelink/commit/b01cb0db985a8c5356b58751460981c97bfffd6f)) +- Reject duplicate store names in StoreManager ([#2533](https://github.com/TraceMachina/nativelink/issues/2533)) - ([2106b1a](https://github.com/TraceMachina/nativelink/commit/2106b1a8481c502e7069b2635f39d83a54b37268)) +- Add free-disk to Redis store tester ([#2543](https://github.com/TraceMachina/nativelink/issues/2543)) - ([4b1de2c](https://github.com/TraceMachina/nativelink/commit/4b1de2cf24a86429480235d4641668531ede4e72)) +- Fix flakey MongoDB test ([#2535](https://github.com/TraceMachina/nativelink/issues/2535)) - ([b8e8171](https://github.com/TraceMachina/nativelink/commit/b8e81711a8a5d3048884ef94667415d4afe52a2a)) + +### ⚙️ Miscellaneous + +- Remove more unused items ([#2561](https://github.com/TraceMachina/nativelink/issues/2561)) - ([8a8be18](https://github.com/TraceMachina/nativelink/commit/8a8be1879d70eceff4caf1b73d6ae63c61fddc5d)) +- Don't update config reference without changes ([#2562](https://github.com/TraceMachina/nativelink/issues/2562)) - ([48d9b96](https://github.com/TraceMachina/nativelink/commit/48d9b961e5403a904708e3394eead2a26df30d81)) +- Tag config regen builds to our bot account ([#2554](https://github.com/TraceMachina/nativelink/issues/2554)) - ([5003676](https://github.com/TraceMachina/nativelink/commit/50036764f6e87fe0518f62eb2ee6b2924f7fadd7)) +- Enforce Vale settings for config as well as markdown ([#2552](https://github.com/TraceMachina/nativelink/issues/2552)) - ([27fb8ad](https://github.com/TraceMachina/nativelink/commit/27fb8adde6554de68d452ef87535822ad4330680)) +- curl 8.5.0-2ubuntu10.11 security update ([#2544](https://github.com/TraceMachina/nativelink/issues/2544)) - ([a8e383a](https://github.com/TraceMachina/nativelink/commit/a8e383ae5e8bd78c8b50fdbad17e7376428a3b3e)) + +### ⬆️ Bumps & Version Updates + +- Upgrade anyhow to 1.0.103 for RUSTSEC-2026-0190 ([#2569](https://github.com/TraceMachina/nativelink/issues/2569)) - ([6675acb](https://github.com/TraceMachina/nativelink/commit/6675acb9ed130ee11f174772a0677500944c8049)) +- Upgrade quick-xml to 0.41 for RUSTSEC-2026-0194 ([#2566](https://github.com/TraceMachina/nativelink/issues/2566)) - ([b173e4b](https://github.com/TraceMachina/nativelink/commit/b173e4be5a65abf56e2c0d620aa7ea23b9107900)) +- Upgrade serde_with to 3.21 for GHSA-7gcf-g7xr-8hxj ([#2559](https://github.com/TraceMachina/nativelink/issues/2559)) - ([b18311f](https://github.com/TraceMachina/nativelink/commit/b18311f46f341bf30e4067a04b9ed93866b9bd46)) +- Update nixpkgs so we can drop our patched renovate ([#2537](https://github.com/TraceMachina/nativelink/issues/2537)) - ([ad0d83d](https://github.com/TraceMachina/nativelink/commit/ad0d83dd80d4f1c88f40468722cfb950bd7e87af)) + ## [1.6.1](https://github.com/TraceMachina/nativelink/compare/v1.6.0..v1.6.1) - 2026-07-08 ### ⛰️ Features diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 8d05b1701..220245f11 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -386,6 +386,44 @@ suggestions invoke it directly: vale somefile ``` +## Contributing to Docs or the Website + +The documentation site (docs.nativelink.com) and the marketing site +(nativelink.com) live in the `web/` directory — a Bun + Turborepo workspace +with two Next.js apps: `web/apps/docs` (the docs, built with Fumadocs) and +`web/apps/web` (the website). This replaces the old `web/platform` layout; +the `bun setup`, `bun docs`, and `bun preview` commands from that era no +longer exist. + +The `web/` workspace is **not** part of the Bazel build — it's built with +Bun only. Install Bun (https://bun.sh, or `pkgs.bun` is available in the nix +dev shell), then: + +```bash +cd web +bun install + +# Live dev server for just the docs at http://localhost:3001/docs +bun dev:docs + +# Production build + preview of the docs app +bun --filter @nativelink/docs build +bun --filter @nativelink/docs start # serves the build on port 3001 +``` + +Docs content is MDX under `web/apps/docs/content/docs/`. Every page needs +`title` and `description` frontmatter; sidebar order comes from each +section's `meta.json` (the top-level `content/docs/meta.json` orders the +sections themselves). Authoring conventions, available MDX components, and +style rules are documented on the site itself under +Contribute → [Working on documentation](https://docs.nativelink.com/contribute/docs) +— which is also the page to update if this workflow changes. + +The configuration reference pages under +`web/apps/docs/content/docs/reference/nativelink-config/` are autogenerated +from the Rust config crate — don't edit them by hand; regenerate with +`bun --filter @nativelink/docs gen:config-reference` from `web/`. + ## Creating releases To keep the release process in line with best practices for open source @@ -475,6 +513,19 @@ most automatically generated changelogs provide. use previous release notes as template by clicking on the "Edit" button on a previous release and copy-pasting the contents into the new release notes. + Attribute every entry to its author. Keep the same category headings as the + changelog and write each line as ` by @author in (#PR) - (short-sha)`, + then close the notes with a `New Contributors` section and a `Full Changelog` + link. To pull the author list and first-time contributors for the range, run: + + ```bash + gh api -X POST repos/TraceMachina/nativelink/releases/generate-notes \ + -f tag_name=v0.x.y -f previous_tag_name=v0.x.z + ``` + + Fold that output into the categorized notes rather than pasting it verbatim, + since the changelog groups commits by type. + Make sure to include migration instructions for all breaking changes. Explicitly list whatever changes you think are worth mentioning as `Major diff --git a/Cargo.lock b/Cargo.lock index dc92c0c58..70926d068 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -97,9 +97,9 @@ dependencies = [ [[package]] name = "anyhow" -version = "1.0.103" +version = "1.0.104" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2a4385e2e34eb35d6b3efe798b9eb88096925d87726c0798709bf56d9ed84af3" +checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470" [[package]] name = "arc-swap" @@ -165,7 +165,7 @@ checksum = "c7c24de15d275a1ecfd47a380fb4d5ec9bfe0933f309ed5e705b775596a3574d" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -176,7 +176,7 @@ checksum = "9035ad2d096bed7955a320ee7e2230574d28fd3c3a0f186cbea1ff3c7eed5dbb" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -586,7 +586,7 @@ checksum = "8d7396fd9500589e62e460e987ecb671bad374934e55ec3b5f498cc7a8a8a7b7" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -721,7 +721,7 @@ checksum = "b9b52dba6a345f3ad2d42ff8d0d63df9d0994cfa29657bf18ffdbf149f78a4f5" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", "tracing", ] @@ -1053,7 +1053,7 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -1323,14 +1323,14 @@ checksum = "f46882e17999c6cc590af592290432be3bce0428cb0d5f8b6715e4dc7b383eb3" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] name = "darling" -version = "0.21.3" +version = "0.23.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9cdf337090841a411e2a7f3deb9187445851f91b309c0c0a29e05f74a00a48c0" +checksum = "25ae13da2f202d56bd7f91c25fba009e7717a1e4a1cc98a76d844b65ae912e9d" dependencies = [ "darling_core", "darling_macro", @@ -1338,27 +1338,26 @@ dependencies = [ [[package]] name = "darling_core" -version = "0.21.3" +version = "0.23.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1247195ecd7e3c85f83c8d2a366e4210d588e802133e1e355180a9870b517ea4" +checksum = "9865a50f7c335f53564bb694ef660825eb8610e0a53d3e11bf1b0d3df31e03b0" dependencies = [ - "fnv", "ident_case", "proc-macro2", "quote", "strsim", - "syn", + "syn 2.0.117", ] [[package]] name = "darling_macro" -version = "0.21.3" +version = "0.23.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d38308df82d1080de0afee5d069fa14b0326a88c14f15c5ccda35b4a6c414c81" +checksum = "ac3984ec7bd6cfa798e62b4a642426a5be0e68f9401cfc2a01e3fa9ea2fcdb8d" dependencies = [ "darling_core", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -1396,7 +1395,7 @@ checksum = "d65d7ce8132b7c0e54497a4d9a55a1c2a0912a0d786cf894472ba818fba45762" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -1407,7 +1406,7 @@ checksum = "ef941ded77d15ca19b40374869ac6000af1c9f2a4c0f3d4c70926287e6364a8f" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -1420,7 +1419,7 @@ dependencies = [ "proc-macro2", "quote", "rustc_version", - "syn", + "syn 2.0.117", ] [[package]] @@ -1441,7 +1440,7 @@ dependencies = [ "proc-macro2", "quote", "rustc_version", - "syn", + "syn 2.0.117", "unicode-xid", ] @@ -1492,7 +1491,7 @@ checksum = "1ac70aa55017e108007fbaf5aa0f54b021c98f92ff8af59d42eda9da96e3dd4f" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -1792,7 +1791,7 @@ checksum = "e835b70203e41293343137df5c0664546da5745f82ec9b84d40be8336958447b" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -2283,7 +2282,7 @@ dependencies = [ "libc", "percent-encoding", "pin-project-lite", - "socket2 0.5.10", + "socket2 0.6.4", "tokio", "tower-service", "tracing", @@ -2543,7 +2542,7 @@ dependencies = [ "quote", "rustc_version", "simd_cesu8", - "syn", + "syn 2.0.117", ] [[package]] @@ -2568,7 +2567,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "38c0b942f458fe50cdac086d2f946512305e5631e720728f2a61aabcd47a6264" dependencies = [ "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -2714,7 +2713,7 @@ dependencies = [ "macro_magic_core", "macro_magic_macros", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -2728,7 +2727,7 @@ dependencies = [ "macro_magic_core_macros", "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -2739,7 +2738,7 @@ checksum = "b02abfe41815b5bd98dbd4260173db2c116dda171dc0fe7838cb206333b83308" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -2750,7 +2749,7 @@ checksum = "73ea28ee64b88876bf45277ed9a5817c1817df061a74f2b988971a12570e5869" dependencies = [ "macro_magic_core", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -2953,7 +2952,7 @@ dependencies = [ "macro_magic", "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -2964,7 +2963,7 @@ checksum = "1d87ecb2933e8aeadb3e3a02b828fed80a7528047e68b4f424523a0981a3a084" [[package]] name = "nativelink" -version = "1.6.1" +version = "1.6.3" dependencies = [ "async-lock", "axum", @@ -2995,7 +2994,7 @@ dependencies = [ [[package]] name = "nativelink-config" -version = "1.6.1" +version = "1.6.3" dependencies = [ "byte-unit", "humantime", @@ -3014,7 +3013,7 @@ dependencies = [ [[package]] name = "nativelink-error" -version = "1.6.1" +version = "1.6.3" dependencies = [ "base64 0.22.1", "mongodb", @@ -3038,16 +3037,16 @@ dependencies = [ [[package]] name = "nativelink-macro" -version = "1.6.1" +version = "1.6.3" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 3.0.1", ] [[package]] name = "nativelink-metric" -version = "1.6.1" +version = "1.6.3" dependencies = [ "async-lock", "nativelink-metric-macro-derive", @@ -3058,16 +3057,16 @@ dependencies = [ [[package]] name = "nativelink-metric-macro-derive" -version = "1.6.1" +version = "1.6.3" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 3.0.1", ] [[package]] name = "nativelink-proto" -version = "1.6.1" +version = "1.6.3" dependencies = [ "derive_more 2.1.0", "prost", @@ -3079,7 +3078,7 @@ dependencies = [ [[package]] name = "nativelink-redis-tester" -version = "1.6.1" +version = "1.6.3" dependencies = [ "either", "nativelink-util", @@ -3092,7 +3091,7 @@ dependencies = [ [[package]] name = "nativelink-scheduler" -version = "1.6.1" +version = "1.6.3" dependencies = [ "async-lock", "async-trait", @@ -3129,7 +3128,7 @@ dependencies = [ [[package]] name = "nativelink-service" -version = "1.6.1" +version = "1.6.3" dependencies = [ "async-lock", "async-trait", @@ -3173,7 +3172,7 @@ dependencies = [ [[package]] name = "nativelink-store" -version = "1.6.1" +version = "1.6.3" dependencies = [ "async-lock", "async-trait", @@ -3247,11 +3246,12 @@ dependencies = [ "webpki-roots 1.0.3", "wincode", "zip", + "zstd", ] [[package]] name = "nativelink-util" -version = "1.6.1" +version = "1.6.3" dependencies = [ "anyhow", "async-trait", @@ -3309,11 +3309,12 @@ dependencies = [ "uuid", "walkdir", "wincode", + "zstd", ] [[package]] name = "nativelink-worker" -version = "1.6.1" +version = "1.6.3" dependencies = [ "async-lock", "async-trait", @@ -3714,7 +3715,7 @@ dependencies = [ "pest_meta", "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -3755,7 +3756,7 @@ checksum = "6e918e4ff8c4549eb882f14b3a4bc8c8bc93de829416eacf579f1207a8fbf861" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -3855,7 +3856,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" dependencies = [ "proc-macro2", - "syn", + "syn 2.0.117", ] [[package]] @@ -3901,7 +3902,7 @@ dependencies = [ "prost", "prost-types", "regex", - "syn", + "syn 2.0.117", "tempfile", ] @@ -3915,7 +3916,7 @@ dependencies = [ "itertools", "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -3929,9 +3930,9 @@ dependencies = [ [[package]] name = "quick-xml" -version = "0.39.4" +version = "0.41.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cdcc8dd4e2f670d309a5f0e83fe36dfdc05af317008fea29144da1a2ac858e5e" +checksum = "e660451e55124f798a69a5af3f49ccfbefbd41910eefd25caf2393e1f3473ec1" dependencies = [ "memchr", "serde", @@ -4134,7 +4135,7 @@ checksum = "b7186006dcb21920990093f30e3dea63b7d6e977bf1256be20c3563a5db070da" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -4488,7 +4489,7 @@ dependencies = [ "proc-macro2", "quote", "serde_derive_internals", - "syn", + "syn 2.0.117", ] [[package]] @@ -4583,7 +4584,7 @@ checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -4594,7 +4595,7 @@ checksum = "18d26a20a969b9e3fdf2fc2d9f21eda6c40e2de84c9408bb5d3b05d499aae711" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -4645,11 +4646,12 @@ dependencies = [ [[package]] name = "serde_with" -version = "3.15.1" +version = "3.21.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aa66c845eee442168b2c8134fec70ac50dc20e760769c8ba0ad1319ca1959b04" +checksum = "76a5c54c7310e7b8b9577c286d7e399ddd876c3e12b3ed917a8aabc4b96e9e8c" dependencies = [ "base64 0.22.1", + "bs58", "chrono", "hex", "indexmap 1.9.3", @@ -4664,14 +4666,14 @@ dependencies = [ [[package]] name = "serde_with_macros" -version = "3.15.1" +version = "3.21.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b91a903660542fced4e99881aa481bdbaec1634568ee02e0b8bd57c64cb38955" +checksum = "84d57bc0c8b9a17920c178daa6bb924850d54a9c97ab45194bb8c17ad66bb660" dependencies = [ "darling", "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -4695,7 +4697,7 @@ checksum = "94e153fc76e1c6a068703d6d29c508a0b15c061c4b7e43da59cc097bc342673c" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -4901,6 +4903,17 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "syn" +version = "3.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5edbec4ed188954a10c12c038215f8ce7606b2d5c973cd8dc43e8795065c5f2f" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + [[package]] name = "sync_wrapper" version = "1.0.2" @@ -4918,7 +4931,7 @@ checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -5009,7 +5022,7 @@ checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -5020,7 +5033,7 @@ checksum = "ebc4ee7f67670e9b64d05fa4253e753e016c6c95ff35b89b7941d6b856dec1d5" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -5131,7 +5144,7 @@ checksum = "385a6cb71ab9ab790c5fe8d67f1645e6c450a7ce006a33de03daa956cf70a496" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -5211,7 +5224,7 @@ dependencies = [ "prettyplease", "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -5236,7 +5249,7 @@ dependencies = [ "prost-build", "prost-types", "quote", - "syn", + "syn 2.0.117", "tempfile", "tonic-build", ] @@ -5320,7 +5333,7 @@ checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -5408,7 +5421,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "04659ddb06c87d233c566112c1c9c5b9e98256d9af50ec3bc9c8327f873a7568" dependencies = [ "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -5434,7 +5447,7 @@ checksum = "3c36781cc0e46a83726d9879608e4cf6c2505237e263a8eb8c24502989cfdb28" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -5451,9 +5464,9 @@ checksum = "562d481066bde0658276a35467c4af00bdc6ee726305698a55b86e61d7ad82bb" [[package]] name = "typespec" -version = "1.0.0" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "21666a31293beab8f41d38c2849ddbc342cd9c7cb4d71a9818868287a8934e53" +checksum = "753a2fe021e407d4fc9ee6f4f0a33403cc306d5c54c4e4ebe1b8cbde0ca052b9" dependencies = [ "base64 0.22.1", "bytes", @@ -5498,7 +5511,7 @@ dependencies = [ "proc-macro2", "quote", "rustc_version", - "syn", + "syn 2.0.117", ] [[package]] @@ -5704,7 +5717,7 @@ dependencies = [ "bumpalo", "proc-macro2", "quote", - "syn", + "syn 2.0.117", "wasm-bindgen-shared", ] @@ -5837,9 +5850,9 @@ dependencies = [ [[package]] name = "wincode" -version = "0.5.4" +version = "0.5.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "37095eb18dd6254c66217edc61a29d83d51f8818de8a2ffe88e4584ad73fb5f9" +checksum = "66d967db7705dc29120bb6e8ce5b5a2e27734ed5976d1c904e95bd238d1c3c5a" dependencies = [ "pastey", "proc-macro2", @@ -5850,14 +5863,14 @@ dependencies = [ [[package]] name = "wincode-derive" -version = "0.4.5" +version = "0.4.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e262d55d1261f31e2cfe49cc6385a421d14d99faa0526bbe3cc1bda0d3005c62" +checksum = "15ab90b719560d0fda79c74550ad1c948d17b118765942838055ebaf34d67071" dependencies = [ "darling", "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -5881,7 +5894,7 @@ checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -5892,7 +5905,7 @@ checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -6197,7 +6210,7 @@ dependencies = [ "heck", "indexmap 2.14.0", "prettyplease", - "syn", + "syn 2.0.117", "wasm-metadata", "wit-bindgen-core", "wit-component", @@ -6213,7 +6226,7 @@ dependencies = [ "prettyplease", "proc-macro2", "quote", - "syn", + "syn 2.0.117", "wit-bindgen-core", "wit-bindgen-rust", ] @@ -6307,7 +6320,7 @@ checksum = "de844c262c8848816172cef550288e7dc6c7b7814b4ee56b3e1553f275f1858e" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", "synstructure", ] @@ -6328,7 +6341,7 @@ checksum = "0b631b19d36a892ab55420c92dbc83ccd79274f25be714855d3074aa71cab639" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -6348,7 +6361,7 @@ checksum = "11532158c46691caf0f2593ea8358fed6bbf68a0315e80aae9bd41fbade684a1" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", "synstructure", ] @@ -6388,7 +6401,7 @@ checksum = "625dc425cab0dca6dc3c3319506e6593dcb08a9f387ea3b284dbd52a92c40555" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 4515689cf..d9f029382 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -11,7 +11,7 @@ resolver = "2" edition = "2024" name = "nativelink" rust-version = "1.93.1" -version = "1.6.1" +version = "1.6.3" [profile.release] lto = true @@ -152,6 +152,7 @@ alloc-instead-of-core = "deny" as-underscore = "deny" await-holding-lock = "deny" bind-instead-of-map = "deny" +cast_possible_wrap = { level = "deny", priority = 1 } collapsible-if = "deny" disallowed-methods = "deny" doc-markdown = "deny" @@ -205,7 +206,6 @@ uninhabited-references = { level = "allow", priority = 1 } # rust-lang/rust # TODO(palfrey): Remove these to get to pedantic. cast_possible_truncation = { level = "deny", priority = 1 } -cast_possible_wrap = { level = "allow", priority = 1 } cast_precision_loss = { level = "allow", priority = 1 } cast_sign_loss = { level = "allow", priority = 1 } large_futures = { level = "allow", priority = 1 } diff --git a/MODULE.bazel b/MODULE.bazel index ce93ef403..e6f269e54 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -1,10 +1,10 @@ module( name = "nativelink", - version = "1.6.1", + version = "1.6.3", compatibility_level = 0, ) -bazel_dep(name = "rules_cc", version = "0.2.18") +bazel_dep(name = "rules_cc", version = "0.2.19") bazel_dep(name = "platforms", version = "1.1.0") bazel_dep(name = "bazel_skylib", version = "1.9.0") bazel_dep(name = "rules_python", version = "2.0.0") @@ -15,7 +15,7 @@ bazel_dep(name = "rules_shell", version = "0.6.1") # via --extra_toolchains, which outranks the MODULE.bazel registration below, # so Nix users keep getting LRE-CC unchanged. Outside Nix (no lre.bazelrc), # these hermetic toolchains replace the host autodetect path. -bazel_dep(name = "llvm", version = "0.7.7") +bazel_dep(name = "llvm", version = "0.8.9") register_toolchains("@llvm//toolchain:all") @@ -50,9 +50,10 @@ register_toolchains( dev_dependency = True, ) -bazel_dep(name = "rules_rs", version = "0.0.76") +bazel_dep(name = "rules_rs", version = "0.0.96") -# Pin rules_rust to the hermeticbuild fork (the same commit rules_rs provisions) +# Pin rules_rust to the hermeticbuild fork +# (see https://github.com/hermeticbuild/rules_rs/blob/v0.0.96/rs/rules_rust.bzl) # so all `@rules_rust//...` references resolve to the patched ruleset. The # bazel_dep + archive_override is the form that keeps `@rules_rust` visible to # subsequent `use_extension(...)` calls in this MODULE.bazel — the alternative @@ -62,15 +63,15 @@ bazel_dep(name = "rules_rs", version = "0.0.76") # The local musl-platforms patch is still applied because the hermeticbuild # fork does not list x86_64/aarch64-unknown-linux-musl as supported triples in # rust/platform/triple_mappings.bzl. -bazel_dep(name = "rules_rust", version = "0.68.1") +bazel_dep(name = "rules_rust", version = "0.71.3") archive_override( module_name = "rules_rust", - integrity = "sha256-HG4cSGKVIoZTn0zpUNKhJbGvFfD2UVPJqKRqgTqLOQQ=", + integrity = "sha256-2Omls/gtvOOdKgmcXazTqrCxf5zegAK6JMSRL/3J51c=", patch_strip = 1, patches = ["//tools:rules_rust-musl-platforms.diff"], - strip_prefix = "rules_rust-cf176d81c12d9c8f6420c7d433b0af0f08d2abb1", + strip_prefix = "rules_rust-f19661ba71f77c993ce1eb082053692610fe4313", urls = [ - "https://github.com/hermeticbuild/rules_rust/archive/cf176d81c12d9c8f6420c7d433b0af0f08d2abb1.tar.gz", + "https://github.com/hermeticbuild/rules_rust/archive/f19661ba71f77c993ce1eb082053692610fe4313.tar.gz", ], ) diff --git a/MODULE.bazel.lock b/MODULE.bazel.lock index 9b7bf7678..0235798ee 100644 --- a/MODULE.bazel.lock +++ b/MODULE.bazel.lock @@ -37,7 +37,6 @@ "https://bcr.bazel.build/modules/bazel_features/1.28.0/MODULE.bazel": "4b4200e6cbf8fa335b2c3f43e1d6ef3e240319c33d43d60cc0fbd4b87ece299d", "https://bcr.bazel.build/modules/bazel_features/1.3.0/MODULE.bazel": "cdcafe83ec318cda34e02948e81d790aab8df7a929cec6f6969f13a489ccecd9", "https://bcr.bazel.build/modules/bazel_features/1.30.0/MODULE.bazel": "a14b62d05969a293b80257e72e597c2da7f717e1e69fa8b339703ed6731bec87", - "https://bcr.bazel.build/modules/bazel_features/1.32.0/MODULE.bazel": "095d67022a58cb20f7e20e1aefecfa65257a222c18a938e2914fd257b5f1ccdc", "https://bcr.bazel.build/modules/bazel_features/1.33.0/MODULE.bazel": "8b8dc9d2a4c88609409c3191165bccec0e4cb044cd7a72ccbe826583303459f6", "https://bcr.bazel.build/modules/bazel_features/1.34.0/MODULE.bazel": "e8475ad7c8965542e0c7aac8af68eb48c4af904be3d614b6aa6274c092c2ea1e", "https://bcr.bazel.build/modules/bazel_features/1.4.1/MODULE.bazel": "e45b6bb2350aff3e442ae1111c555e27eac1d915e77775f6fdc4b351b758b5d7", @@ -45,7 +44,9 @@ "https://bcr.bazel.build/modules/bazel_features/1.42.1/MODULE.bazel": "275a59b5406ff18c01739860aa70ad7ccb3cfb474579411decca11c93b951080", "https://bcr.bazel.build/modules/bazel_features/1.43.0/MODULE.bazel": "defa2226f06ba20550d6548c3a2ea2a7929634437a52973869c20c225450eb91", "https://bcr.bazel.build/modules/bazel_features/1.45.0/MODULE.bazel": "7daec6d87ab0703417486d4cb948af0b06f55d4d7c08cbb5978c80e79b538edf", - "https://bcr.bazel.build/modules/bazel_features/1.45.0/source.json": "635e4536e09ff125b8972e0fa239c135fde5f18701f7d5115680560651dfb41d", + "https://bcr.bazel.build/modules/bazel_features/1.47.0/MODULE.bazel": "e34df3cb35b1684cfa69923a61ae3803595babd3942cd306a488d51400886b30", + "https://bcr.bazel.build/modules/bazel_features/1.50.0/MODULE.bazel": "2083ef9c7a469f520890483ccf8e0189d6e71e2117e7752e15e6554433d5ae3e", + "https://bcr.bazel.build/modules/bazel_features/1.50.0/source.json": "e0ee3debde2789ff56e4452e612d126925ba9ab64d4bde79c67f099d2902df9b", "https://bcr.bazel.build/modules/bazel_features/1.9.0/MODULE.bazel": "885151d58d90d8d9c811eb75e3288c11f850e1d6b481a8c9f766adee4712358b", "https://bcr.bazel.build/modules/bazel_features/1.9.1/MODULE.bazel": "8f679097876a9b609ad1f60249c49d68bfab783dd9be012faf9d82547b14815a", "https://bcr.bazel.build/modules/bazel_lib/3.0.0/MODULE.bazel": "22b70b80ac89ad3f3772526cd9feee2fa412c2b01933fea7ed13238a448d370d", @@ -69,6 +70,8 @@ "https://bcr.bazel.build/modules/bazel_skylib/1.9.0/source.json": "7ad77c1e8c1b84222d9b3f3cae016a76639435744c19330b0b37c0a3c9da7dc0", "https://bcr.bazel.build/modules/buildozer/8.5.1/MODULE.bazel": "a35d9561b3fc5b18797c330793e99e3b834a473d5fbd3d7d7634aafc9bdb6f8f", "https://bcr.bazel.build/modules/buildozer/8.5.1/source.json": "e3386e6ff4529f2442800dee47ad28d3e6487f36a1f75ae39ae56c70f0cd2fbd", + "https://bcr.bazel.build/modules/bzip2/1.0.8.bcr.3/MODULE.bazel": "29ecf4babfd3c762be00d7573c288c083672ab60e79c833ff7f49ee662e54471", + "https://bcr.bazel.build/modules/bzip2/1.0.8.bcr.3/source.json": "8be4a3ef2599693f759e5c0990a4cc5a246ac08db4c900a38f852ba25b5c39be", "https://bcr.bazel.build/modules/gawk/5.3.2.bcr.3/MODULE.bazel": "f1b7bb2dd53e8f2ef984b39485ec8a44e9076dda5c4b8efd2fb4c6a6e856a31d", "https://bcr.bazel.build/modules/gawk/5.3.2.bcr.3/source.json": "ebe931bfe362e4b41e59ee00a528db6074157ff2ced92eb9e970acab2e1089c9", "https://bcr.bazel.build/modules/google_benchmark/1.8.2/MODULE.bazel": "a70cf1bba851000ba93b58ae2f6d76490a9feb74192e57ab8e8ff13c34ec50cb", @@ -78,12 +81,16 @@ "https://bcr.bazel.build/modules/googletest/1.15.2/MODULE.bazel": "6de1edc1d26cafb0ea1a6ab3f4d4192d91a312fd2d360b63adaa213cd00b2108", "https://bcr.bazel.build/modules/googletest/1.17.0/MODULE.bazel": "dbec758171594a705933a29fcf69293d2468c49ec1f2ebca65c36f504d72df46", "https://bcr.bazel.build/modules/googletest/1.17.0/source.json": "38e4454b25fc30f15439c0378e57909ab1fd0a443158aa35aec685da727cd713", + "https://bcr.bazel.build/modules/hermetic_launcher/0.0.8/MODULE.bazel": "3be7b0faca6f1e69e89197999e0b01ce058c42f3e764ef028466f7e0ff77c761", + "https://bcr.bazel.build/modules/hermetic_launcher/0.0.8/source.json": "8403718636114198fca6ea04b437fa001ac7167b4d485102defb0a6f7d11bc08", "https://bcr.bazel.build/modules/jsoncpp/1.9.5/MODULE.bazel": "31271aedc59e815656f5736f282bb7509a97c7ecb43e927ac1a37966e0578075", "https://bcr.bazel.build/modules/jsoncpp/1.9.6/MODULE.bazel": "2f8d20d3b7d54143213c4dfc3d98225c42de7d666011528dc8fe91591e2e17b0", "https://bcr.bazel.build/modules/jsoncpp/1.9.6/source.json": "a04756d367a2126c3541682864ecec52f92cdee80a35735a3cb249ce015ca000", "https://bcr.bazel.build/modules/libpfm/4.11.0/MODULE.bazel": "45061ff025b301940f1e30d2c16bea596c25b176c8b6b3087e92615adbd52902", "https://bcr.bazel.build/modules/llvm/0.7.7/MODULE.bazel": "0eeaf1814feca77abc7af3523e2b9d3735f92e2583043f8d6a2cc0fb5f479a28", - "https://bcr.bazel.build/modules/llvm/0.7.7/source.json": "7c8910307329462a21b7bdcc52710360da3de8284738dca52241bb15302a00dc", + "https://bcr.bazel.build/modules/llvm/0.8.3/MODULE.bazel": "dbc1bc13171f8f9980a75524883abef1c491613e8d8d805cd4ffa3767ba9708e", + "https://bcr.bazel.build/modules/llvm/0.8.9/MODULE.bazel": "9e35ff5bcac996f9edc1b44b8f8baa58c7855e742c5608b07d925dcdbe642100", + "https://bcr.bazel.build/modules/llvm/0.8.9/source.json": "424b907b38875f1346b65946e5e482b202b842bd3a570abbe07047d2bc7d667b", "https://bcr.bazel.build/modules/nlohmann_json/3.6.1/MODULE.bazel": "6f7b417dcc794d9add9e556673ad25cb3ba835224290f4f848f8e2db1e1fca74", "https://bcr.bazel.build/modules/package_metadata/0.0.3/MODULE.bazel": "77890552ecea9e284b5424c9de827a58099348763a4359e975c359a83d4faa83", "https://bcr.bazel.build/modules/package_metadata/0.0.7/MODULE.bazel": "7adb03933fc8401f495800cf4eafcff0edc6da0ff55c7db223ef69d19f689486", @@ -138,7 +145,8 @@ "https://bcr.bazel.build/modules/rules_cc/0.2.13/MODULE.bazel": "eecdd666eda6be16a8d9dc15e44b5c75133405e820f620a234acc4b1fdc5aa37", "https://bcr.bazel.build/modules/rules_cc/0.2.17/MODULE.bazel": "1849602c86cb60da8613d2de887f9566a6d354a6df6d7009f9d04a14402f9a84", "https://bcr.bazel.build/modules/rules_cc/0.2.18/MODULE.bazel": "4460ec36adc8f722a6a2a4ac9374cb91f2acebadaa93fc37966129afb3dece87", - "https://bcr.bazel.build/modules/rules_cc/0.2.18/source.json": "abad668ff2fd63ada1ac49bf386d37e27048b89a3465a6fd968bb832b00a09d3", + "https://bcr.bazel.build/modules/rules_cc/0.2.19/MODULE.bazel": "d5e0f05b63273281a16654eb6b1a8742a75ec153ac8b4f0419949d6e401e46f0", + "https://bcr.bazel.build/modules/rules_cc/0.2.19/source.json": "1ef48cdbd7aa6238015189b582d3d74ef0cbea3cb3e2cb259d782463f570c14a", "https://bcr.bazel.build/modules/rules_cc/0.2.4/MODULE.bazel": "1ff1223dfd24f3ecf8f028446d4a27608aa43c3f41e346d22838a4223980b8cc", "https://bcr.bazel.build/modules/rules_cc/0.2.8/MODULE.bazel": "f1df20f0bf22c28192a794f29b501ee2018fa37a3862a1a2132ae2940a23a642", "https://bcr.bazel.build/modules/rules_foreign_cc/0.9.0/MODULE.bazel": "c9e8c682bf75b0e7c704166d79b599f93b72cfca5ad7477df596947891feeef6", @@ -192,8 +200,9 @@ "https://bcr.bazel.build/modules/rules_python/1.7.0/MODULE.bazel": "d01f995ecd137abf30238ad9ce97f8fc3ac57289c8b24bd0bf53324d937a14f8", "https://bcr.bazel.build/modules/rules_python/2.0.0/MODULE.bazel": "1459089e2d4194d2a49e07896f5334fb230a8f2966ae945b1f793bef87a292fd", "https://bcr.bazel.build/modules/rules_python/2.0.0/source.json": "b8e25661f58c573e5e27af21295867e87766e89211f326fcb84034e6e6b6794b", - "https://bcr.bazel.build/modules/rules_rs/0.0.76/MODULE.bazel": "461dcf664f368fdc921f67ea20ec1bc78c73f65a0a20b6e2a6d4b1c77fbde8c1", - "https://bcr.bazel.build/modules/rules_rs/0.0.76/source.json": "167eb1122e0f74848fc995b581061155dda1dfd600a38c253a85ef46d0523221", + "https://bcr.bazel.build/modules/rules_rs/0.0.82/MODULE.bazel": "fb58eade192a4b957f6e697bdd665861e80acbd504eb1129e84c57cb54f8cba4", + "https://bcr.bazel.build/modules/rules_rs/0.0.96/MODULE.bazel": "678fdcee5a9847611276770eab47190dda04a41973b7bf6037e50a3f98e49e09", + "https://bcr.bazel.build/modules/rules_rs/0.0.96/source.json": "2b52d3d209324bd4aaa4896aa0e8b28d0123856b4ade32ebd7bf24b4fded6f12", "https://bcr.bazel.build/modules/rules_shell/0.2.0/MODULE.bazel": "fda8a652ab3c7d8fee214de05e7a9916d8b28082234e8d2c0094505c5268ed3c", "https://bcr.bazel.build/modules/rules_shell/0.3.0/MODULE.bazel": "de4402cd12f4cc8fda2354fce179fdb068c0b9ca1ec2d2b17b3e21b24c1a937b", "https://bcr.bazel.build/modules/rules_shell/0.4.1/MODULE.bazel": "00e501db01bbf4e3e1dd1595959092c2fadf2087b2852d3f553b5370f5633592", @@ -209,13 +218,16 @@ "https://bcr.bazel.build/modules/stardoc/0.7.2/MODULE.bazel": "fc152419aa2ea0f51c29583fab1e8c99ddefd5b3778421845606ee628629e0e5", "https://bcr.bazel.build/modules/swift_argument_parser/1.3.1.1/MODULE.bazel": "5e463fbfba7b1701d957555ed45097d7f984211330106ccd1352c6e0af0dcf91", "https://bcr.bazel.build/modules/swift_argument_parser/1.3.1.2/MODULE.bazel": "75aab2373a4bbe2a1260b9bf2a1ebbdbf872d3bd36f80bff058dccd82e89422f", + "https://bcr.bazel.build/modules/tar.bzl/0.10.4/MODULE.bazel": "e8f9ff79199e8d9eaad7f1b0a77ad74b30bb82d794b87d8ca942bead5de83ae9", + "https://bcr.bazel.build/modules/tar.bzl/0.10.4/source.json": "20143442376c03426f6135292ba02d825cb75308aa47e6bf42dd4cc5a435c2ff", "https://bcr.bazel.build/modules/tar.bzl/0.9.0/MODULE.bazel": "452a22d7f02b1c9d7a22ab25edf20f46f3e1101f0f67dc4bfbf9a474ddf02445", - "https://bcr.bazel.build/modules/tar.bzl/0.9.0/source.json": "c732760a374831a2cf5b08839e4be75017196b4d796a5aa55235272ee17cd839", "https://bcr.bazel.build/modules/toolchains_protoc/0.4.3/MODULE.bazel": "54daf5468a9c3e52f6c8a96c8e0b867f7b30029dfe1e74f5a59bf081921d91a3", "https://bcr.bazel.build/modules/toolchains_protoc/0.4.3/source.json": "fbf3886395e08c407caca84f92f8c9ad92b05ce126a94883def1e150edd6b417", "https://bcr.bazel.build/modules/upb/0.0.0-20220923-a547704/MODULE.bazel": "7298990c00040a0e2f121f6c32544bab27d4452f80d9ce51349b1a28f3005c43", "https://bcr.bazel.build/modules/with_cfg.bzl/0.12.0/MODULE.bazel": "b573395fe63aef4299ba095173e2f62ccfee5ad9bbf7acaa95dba73af9fc2b38", "https://bcr.bazel.build/modules/with_cfg.bzl/0.12.0/source.json": "3f3fbaeafecaf629877ad152a2c9def21f8d330d91aa94c5dc75bbb98c10b8b8", + "https://bcr.bazel.build/modules/xz/5.4.5.bcr.8/MODULE.bazel": "e48a69bd54053c2ec5fffc2a29fb70122afd3e83ab6c07068f63bc6553fa57cc", + "https://bcr.bazel.build/modules/xz/5.4.5.bcr.8/source.json": "bd7e928ccd63505b44f4784f7bbf12cc11f9ff23bf3ca12ff2c91cd74846099e", "https://bcr.bazel.build/modules/zlib/1.2.11/MODULE.bazel": "07b389abc85fdbca459b69e2ec656ae5622873af3f845e1c9d80fe179f3effa0", "https://bcr.bazel.build/modules/zlib/1.3.1.bcr.5/MODULE.bazel": "eec517b5bbe5492629466e11dae908d043364302283de25581e3eb944326c4ca", "https://bcr.bazel.build/modules/zlib/1.3.1.bcr.5/source.json": "22bc55c47af97246cfc093d0acf683a7869377de362b5d1c552c2c2e16b7a806", @@ -225,8 +237,8 @@ "moduleExtensions": { "@@aspect_tools_telemetry+//:extension.bzl%telemetry": { "general": { - "bzlTransitiveDigest": "MePriaXmQNSqUfE+YQvEtzBc2bU1mjITINIffFiZYgo=", - "usagesDigest": "ibiT5THLv67F8GNR92fjXCqmyYBFE4ZBphBLqWLTFBI=", + "bzlTransitiveDigest": "cl5A2O84vDL6Tt+Qga8FCj1DUDGqn+e7ly5rZ+4xvcc=", + "usagesDigest": "Miy0EWu0H7wJMzfdekpzuz3TBOzJz0l6aNIq2pL6k2g=", "recordedInputs": [ "REPO_MAPPING:aspect_tools_telemetry+,bazel_lib bazel_lib+", "REPO_MAPPING:aspect_tools_telemetry+,bazel_skylib bazel_skylib+" @@ -236,7 +248,7 @@ "repoRuleId": "@@aspect_tools_telemetry+//:extension.bzl%tel_repository", "attributes": { "deps": { - "rules_rs": "0.0.76", + "rules_rs": "0.0.96", "aspect_tools_telemetry": "0.3.3" } } @@ -246,7 +258,7 @@ }, "@@local-remote-execution+//rust:extension.bzl%lre_rs": { "general": { - "bzlTransitiveDigest": "fWyaTPZjcl+fLiFO5qQKGngxK6FHnIytsGV/9gWcS+w=", + "bzlTransitiveDigest": "uG+JZDxQyPELNihTm54EkV+15PXyp8wI4w/Peu5bXa4=", "usagesDigest": "R9P8gnprFTJ/SsXLiQE5qx+Sqsa8YOFAOmRUyDO3tLY=", "recordedInputs": [ "REPO_MAPPING:local-remote-execution+,bazel_tools bazel_tools" @@ -313,7 +325,7 @@ }, "@@protobuf+//python/dist:system_python.bzl%system_python_extension": { "general": { - "bzlTransitiveDigest": "yCL9NdIIefY+80P+oAIF4HmY/SRaPmtueyPEL7kVOB8=", + "bzlTransitiveDigest": "pmsA+awieucfllLc2n7k8xEoPp0i5LF9Hw6mGX0cqSQ=", "usagesDigest": "A+RWmbKdBBwZcBbNGNvfPbqG2vYZRjVrFp6x1iRUrAk=", "recordedInputs": [], "generatedRepoSpecs": { @@ -328,11 +340,11 @@ }, "@@pybind11_bazel+//:internal_configure.bzl%internal_configure_extension": { "general": { - "bzlTransitiveDigest": "/zSuBEqJ9BGcQV0BWE0KXHwECZB0L0FQ7oGDb2maycA=", + "bzlTransitiveDigest": "b+RP7Sgl8KN0VHamrgTqzGLuYPcQ/Mo4ptNkkHUIIlA=", "usagesDigest": "D1r3lfzMuUBFxgG8V6o0bQTLMk3GkaGOaPzw53wrwyw=", "recordedInputs": [ "REPO_MAPPING:pybind11_bazel+,bazel_tools bazel_tools", - "FILE:@@pybind11_bazel+//MODULE.bazel 45d98c531db5f8430f847b195357dca36665d9c926cbbee77a85105fc6865d27" + "FILE:@@pybind11_bazel+//MODULE.bazel e6f4c20442eaa7c90d7190d8dc539d0ab422f95c65a57cc59562170c58ae3d34" ], "generatedRepoSpecs": { "pybind11": { @@ -350,7 +362,7 @@ }, "@@rules_kotlin+//src/main/starlark/core/repositories:bzlmod_setup.bzl%rules_kotlin_extensions": { "general": { - "bzlTransitiveDigest": "76FMguAa6TZAMN5inJ4/C9SZ0VlZnKOlVnlRe9bgvd8=", + "bzlTransitiveDigest": "Ga4z8lQy1YQ5rAMy+dOl0dqcCEBnYNCXku8x3YQmDZI=", "usagesDigest": "QI2z8ZUR+mqtbwsf2fLqYdJAkPOHdOV+tF2yVAUgRzw=", "recordedInputs": [ "REPO_MAPPING:rules_kotlin+,bazel_tools bazel_tools" @@ -407,7 +419,7 @@ }, "@@rules_python+//python/uv:uv.bzl%uv": { "general": { - "bzlTransitiveDigest": "FweP0zel1F17xmT6aBZ3Itw1ogP5QPaaka1Bzndy0Wk=", + "bzlTransitiveDigest": "I8FPZMevE2oI/peSpMBRVIN++WOtfjtJVjbPsBZQ87A=", "usagesDigest": "DwZ4Bamg/skxdi0sa765qXLDi4cL9PQbmLkE7jwQKVU=", "recordedInputs": [ "REPO_MAPPING:rules_python+,bazel_tools bazel_tools", @@ -434,106 +446,6 @@ } } } - }, - "@@rules_rust+//crate_universe/private:internal_extensions.bzl%cu_nr": { - "general": { - "bzlTransitiveDigest": "A+ROJOpIeiQkfLxK27VU3L2WCGCdsciDVdpEFctkd1w=", - "usagesDigest": "w6DeRbiDSXRVPZPJF6BTEEe4fMh1OiL8grDVrOA0M98=", - "recordedInputs": [ - "REPO_MAPPING:bazel_features+,bazel_features_globals bazel_features++version_extension+bazel_features_globals", - "REPO_MAPPING:bazel_features+,bazel_features_version bazel_features++version_extension+bazel_features_version", - "REPO_MAPPING:rules_cc+,bazel_skylib bazel_skylib+", - "REPO_MAPPING:rules_cc+,bazel_tools bazel_tools", - "REPO_MAPPING:rules_cc+,cc_compatibility_proxy rules_cc++compatibility_proxy+cc_compatibility_proxy", - "REPO_MAPPING:rules_cc+,platforms platforms", - "REPO_MAPPING:rules_cc+,rules_cc rules_cc+", - "REPO_MAPPING:rules_cc++compatibility_proxy+cc_compatibility_proxy,rules_cc rules_cc+", - "REPO_MAPPING:rules_rust+,bazel_features bazel_features+", - "REPO_MAPPING:rules_rust+,bazel_skylib bazel_skylib+", - "REPO_MAPPING:rules_rust+,bazel_tools bazel_tools", - "REPO_MAPPING:rules_rust+,cargo_bazel_bootstrap rules_rust++cu_nr+cargo_bazel_bootstrap", - "REPO_MAPPING:rules_rust+,cui rules_rust++cu+cui", - "REPO_MAPPING:rules_rust+,rrc rules_rust++i2+rrc", - "REPO_MAPPING:rules_rust+,rules_cc rules_cc+", - "REPO_MAPPING:rules_rust+,rules_rust rules_rust+" - ], - "generatedRepoSpecs": { - "cargo_bazel_bootstrap": { - "repoRuleId": "@@rules_rust+//cargo/private:cargo_bootstrap.bzl%cargo_bootstrap_repository", - "attributes": { - "srcs": [ - "@@rules_rust+//crate_universe:src/api.rs", - "@@rules_rust+//crate_universe:src/api/lockfile.rs", - "@@rules_rust+//crate_universe:src/cli.rs", - "@@rules_rust+//crate_universe:src/cli/generate.rs", - "@@rules_rust+//crate_universe:src/cli/query.rs", - "@@rules_rust+//crate_universe:src/cli/render.rs", - "@@rules_rust+//crate_universe:src/cli/splice.rs", - "@@rules_rust+//crate_universe:src/cli/vendor.rs", - "@@rules_rust+//crate_universe:src/config.rs", - "@@rules_rust+//crate_universe:src/context.rs", - "@@rules_rust+//crate_universe:src/context/crate_context.rs", - "@@rules_rust+//crate_universe:src/context/platforms.rs", - "@@rules_rust+//crate_universe:src/lib.rs", - "@@rules_rust+//crate_universe:src/lockfile.rs", - "@@rules_rust+//crate_universe:src/main.rs", - "@@rules_rust+//crate_universe:src/metadata.rs", - "@@rules_rust+//crate_universe:src/metadata/cargo_bin.rs", - "@@rules_rust+//crate_universe:src/metadata/cargo_tree_resolver.rs", - "@@rules_rust+//crate_universe:src/metadata/cargo_tree_rustc_wrapper.bat", - "@@rules_rust+//crate_universe:src/metadata/cargo_tree_rustc_wrapper.sh", - "@@rules_rust+//crate_universe:src/metadata/dependency.rs", - "@@rules_rust+//crate_universe:src/metadata/metadata_annotation.rs", - "@@rules_rust+//crate_universe:src/rendering.rs", - "@@rules_rust+//crate_universe:src/rendering/template_engine.rs", - "@@rules_rust+//crate_universe:src/rendering/templates/module_bzl.j2", - "@@rules_rust+//crate_universe:src/rendering/templates/partials/header.j2", - "@@rules_rust+//crate_universe:src/rendering/templates/partials/module/aliases_map.j2", - "@@rules_rust+//crate_universe:src/rendering/templates/partials/module/deps_map.j2", - "@@rules_rust+//crate_universe:src/rendering/templates/partials/module/repo_git.j2", - "@@rules_rust+//crate_universe:src/rendering/templates/partials/module/repo_http.j2", - "@@rules_rust+//crate_universe:src/rendering/templates/vendor_module.j2", - "@@rules_rust+//crate_universe:src/rendering/verbatim/alias_rules.bzl", - "@@rules_rust+//crate_universe:src/select.rs", - "@@rules_rust+//crate_universe:src/splicing.rs", - "@@rules_rust+//crate_universe:src/splicing/cargo_config.rs", - "@@rules_rust+//crate_universe:src/splicing/crate_index_lookup.rs", - "@@rules_rust+//crate_universe:src/splicing/splicer.rs", - "@@rules_rust+//crate_universe:src/test.rs", - "@@rules_rust+//crate_universe:src/utils.rs", - "@@rules_rust+//crate_universe:src/utils/starlark.rs", - "@@rules_rust+//crate_universe:src/utils/starlark/glob.rs", - "@@rules_rust+//crate_universe:src/utils/starlark/label.rs", - "@@rules_rust+//crate_universe:src/utils/starlark/select.rs", - "@@rules_rust+//crate_universe:src/utils/starlark/select_dict.rs", - "@@rules_rust+//crate_universe:src/utils/starlark/select_list.rs", - "@@rules_rust+//crate_universe:src/utils/starlark/select_scalar.rs", - "@@rules_rust+//crate_universe:src/utils/starlark/select_set.rs", - "@@rules_rust+//crate_universe:src/utils/starlark/serialize.rs", - "@@rules_rust+//crate_universe:src/utils/starlark/target_compatible_with.rs", - "@@rules_rust+//crate_universe:src/utils/symlink.rs", - "@@rules_rust+//crate_universe:src/utils/target_triple.rs" - ], - "binary": "cargo-bazel", - "cargo_lockfile": "@@rules_rust+//crate_universe:Cargo.lock", - "cargo_toml": "@@rules_rust+//crate_universe:Cargo.toml", - "version": "1.95.0", - "rust_toolchain_cargo_template": "@rust_host_tools//:bin/{tool}", - "rust_toolchain_rustc_template": "@rust_host_tools//:bin/{tool}", - "compressed_windows_toolchain_names": false, - "timeout": 900 - } - } - }, - "moduleExtensionMetadata": { - "explicitRootModuleDirectDeps": [ - "cargo_bazel_bootstrap" - ], - "explicitRootModuleDirectDevDeps": [], - "useAllRepos": "NO", - "reproducible": false - } - } } }, "facts": { @@ -773,6 +685,8 @@ "adler2_2.0.1": "{\"dependencies\":[{\"name\":\"core\",\"optional\":true,\"package\":\"rustc-std-workspace-core\",\"req\":\"^1.0.0\"}],\"features\":{\"default\":[\"std\"],\"rustc-dep-of-std\":[\"core\"],\"std\":[]}}", "ahash_0.8.12": "{\"dependencies\":[{\"name\":\"cfg-if\",\"req\":\"^1.0\"},{\"name\":\"const-random\",\"optional\":true,\"req\":\"^0.1.17\"},{\"features\":[\"html_reports\"],\"kind\":\"dev\",\"name\":\"criterion\",\"req\":\"^0.3.2\"},{\"kind\":\"dev\",\"name\":\"fnv\",\"req\":\"^1.0.5\"},{\"kind\":\"dev\",\"name\":\"fxhash\",\"req\":\"^0.2.1\"},{\"name\":\"getrandom\",\"optional\":true,\"req\":\"^0.3.1\"},{\"kind\":\"dev\",\"name\":\"hashbrown\",\"req\":\"^0.14.3\"},{\"kind\":\"dev\",\"name\":\"hex\",\"req\":\"^0.4.2\"},{\"kind\":\"dev\",\"name\":\"no-panic\",\"req\":\"^0.1.10\"},{\"default_features\":false,\"features\":[\"alloc\"],\"name\":\"once_cell\",\"req\":\"^1.18.0\",\"target\":\"cfg(not(all(target_arch = \\\"arm\\\", target_os = \\\"none\\\")))\"},{\"kind\":\"dev\",\"name\":\"pcg-mwc\",\"req\":\"^0.2.1\"},{\"name\":\"portable-atomic\",\"optional\":true,\"req\":\"^1.0.0\"},{\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.8.5\"},{\"kind\":\"dev\",\"name\":\"seahash\",\"req\":\"^4.0\"},{\"name\":\"serde\",\"optional\":true,\"req\":\"^1.0.117\"},{\"kind\":\"dev\",\"name\":\"serde_json\",\"req\":\"^1.0.59\"},{\"kind\":\"dev\",\"name\":\"smallvec\",\"req\":\"^1.13.1\"},{\"kind\":\"build\",\"name\":\"version_check\",\"req\":\"^0.9.4\"},{\"default_features\":false,\"features\":[\"simd\"],\"name\":\"zerocopy\",\"req\":\"^0.8.24\"}],\"features\":{\"atomic-polyfill\":[\"dep:portable-atomic\",\"once_cell/critical-section\"],\"compile-time-rng\":[\"const-random\"],\"default\":[\"std\",\"runtime-rng\"],\"nightly-arm-aes\":[],\"no-rng\":[],\"runtime-rng\":[\"getrandom\"],\"std\":[]}}", "aho-corasick_1.1.4": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"doc-comment\",\"req\":\"^0.3.3\"},{\"name\":\"log\",\"optional\":true,\"req\":\"^0.4.17\"},{\"default_features\":false,\"name\":\"memchr\",\"optional\":true,\"req\":\"^2.4.0\"}],\"features\":{\"default\":[\"std\",\"perf-literal\"],\"logging\":[\"dep:log\"],\"perf-literal\":[\"dep:memchr\"],\"std\":[\"memchr?/std\"]}}", + "alloc-no-stdlib_2.0.4": "{\"dependencies\":[],\"features\":{\"unsafe\":[]}}", + "alloc-stdlib_0.2.2": "{\"dependencies\":[{\"name\":\"alloc-no-stdlib\",\"req\":\"^2.0.4\"}],\"features\":{\"unsafe\":[]}}", "allocator-api2_0.2.21": "{\"dependencies\":[{\"name\":\"serde\",\"optional\":true,\"req\":\"^1.0\"}],\"features\":{\"alloc\":[],\"default\":[\"std\"],\"fresh-rust\":[],\"nightly\":[],\"std\":[\"alloc\"]}}", "android_system_properties_0.1.5": "{\"dependencies\":[{\"name\":\"libc\",\"req\":\"^0.2.126\"}],\"features\":{}}", "anstream_0.6.21": "{\"dependencies\":[{\"name\":\"anstyle\",\"req\":\"^1.0.0\"},{\"name\":\"anstyle-parse\",\"req\":\"^0.2.0\"},{\"name\":\"anstyle-query\",\"optional\":true,\"req\":\"^1.0.0\"},{\"name\":\"anstyle-wincon\",\"optional\":true,\"req\":\"^3.0.5\",\"target\":\"cfg(windows)\"},{\"name\":\"colorchoice\",\"req\":\"^1.0.0\"},{\"kind\":\"dev\",\"name\":\"divan\",\"req\":\"^0.1.16\"},{\"name\":\"is_terminal_polyfill\",\"req\":\"^1.48\"},{\"kind\":\"dev\",\"name\":\"lexopt\",\"req\":\"^0.3.1\"},{\"kind\":\"dev\",\"name\":\"owo-colors\",\"req\":\"^4.0.0\"},{\"kind\":\"dev\",\"name\":\"proptest\",\"req\":\"^1.7.0\"},{\"kind\":\"dev\",\"name\":\"strip-ansi-escapes\",\"req\":\"^0.2.1\"},{\"name\":\"utf8parse\",\"req\":\"^0.2.2\"}],\"features\":{\"auto\":[\"dep:anstyle-query\"],\"default\":[\"auto\",\"wincon\"],\"test\":[],\"wincon\":[\"dep:anstyle-wincon\"]}}", @@ -786,11 +700,12 @@ "anstyle_1.0.13": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"lexopt\",\"req\":\"^0.3.0\"},{\"kind\":\"dev\",\"name\":\"snapbox\",\"req\":\"^0.6.5\"}],\"features\":{\"default\":[\"std\"],\"std\":[]}}", "anstyle_1.0.14": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"lexopt\",\"req\":\"^0.3.1\"},{\"kind\":\"dev\",\"name\":\"snapbox\",\"req\":\"^0.6.23\"}],\"features\":{\"default\":[\"std\"],\"std\":[]}}", "anyhow_1.0.102": "{\"dependencies\":[{\"default_features\":false,\"kind\":\"dev\",\"name\":\"futures\",\"req\":\"^0.3\"},{\"kind\":\"dev\",\"name\":\"rustversion\",\"req\":\"^1.0.6\"},{\"features\":[\"full\"],\"kind\":\"dev\",\"name\":\"syn\",\"req\":\"^2.0\"},{\"kind\":\"dev\",\"name\":\"thiserror\",\"req\":\"^2\"},{\"features\":[\"diff\"],\"kind\":\"dev\",\"name\":\"trybuild\",\"req\":\"^1.0.108\"}],\"features\":{\"backtrace\":[],\"default\":[\"std\"],\"std\":[]}}", - "anyhow_1.0.103": "{\"dependencies\":[{\"default_features\":false,\"kind\":\"dev\",\"name\":\"futures\",\"req\":\"^0.3\"},{\"kind\":\"dev\",\"name\":\"rustversion\",\"req\":\"^1.0.6\"},{\"features\":[\"full\"],\"kind\":\"dev\",\"name\":\"syn\",\"req\":\"^2.0\"},{\"kind\":\"dev\",\"name\":\"thiserror\",\"req\":\"^2\"},{\"features\":[\"diff\"],\"kind\":\"dev\",\"name\":\"trybuild\",\"req\":\"^1.0.108\"}],\"features\":{\"backtrace\":[],\"default\":[\"std\"],\"std\":[]}}", + "anyhow_1.0.104": "{\"dependencies\":[{\"default_features\":false,\"kind\":\"dev\",\"name\":\"futures\",\"req\":\"^0.3\"},{\"kind\":\"dev\",\"name\":\"rustversion\",\"req\":\"^1.0.6\"},{\"features\":[\"full\"],\"kind\":\"dev\",\"name\":\"syn\",\"req\":\"^3\"},{\"kind\":\"dev\",\"name\":\"thiserror\",\"req\":\"^2\"},{\"features\":[\"diff\"],\"kind\":\"dev\",\"name\":\"trybuild\",\"req\":\"^1.0.108\"}],\"features\":{\"backtrace\":[],\"default\":[\"std\"],\"std\":[]}}", "arc-swap_1.7.1": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"adaptive-barrier\",\"req\":\"~1\"},{\"kind\":\"dev\",\"name\":\"criterion\",\"req\":\"~0.5\"},{\"kind\":\"dev\",\"name\":\"crossbeam-utils\",\"req\":\"~0.8\"},{\"kind\":\"dev\",\"name\":\"itertools\",\"req\":\"^0.12\"},{\"kind\":\"dev\",\"name\":\"num_cpus\",\"req\":\"~1\"},{\"kind\":\"dev\",\"name\":\"once_cell\",\"req\":\"~1\"},{\"kind\":\"dev\",\"name\":\"parking_lot\",\"req\":\"~0.12\"},{\"kind\":\"dev\",\"name\":\"proptest\",\"req\":\"^1\"},{\"features\":[\"rc\"],\"name\":\"serde\",\"optional\":true,\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"serde_derive\",\"req\":\"^1.0.130\"},{\"kind\":\"dev\",\"name\":\"serde_test\",\"req\":\"^1.0.130\"}],\"features\":{\"experimental-strategies\":[],\"experimental-thread-local\":[],\"internal-test-strategies\":[],\"weak\":[]}}", "arcstr_1.2.0": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"loom\",\"req\":\"^0.7.1\",\"target\":\"cfg(loom)\"},{\"default_features\":false,\"name\":\"serde\",\"optional\":true,\"req\":\"^1\"},{\"default_features\":false,\"kind\":\"dev\",\"name\":\"serde_test\",\"req\":\"^1\"}],\"features\":{\"default\":[\"substr\"],\"std\":[],\"substr\":[],\"substr-usize-indices\":[\"substr\"]}}", "arrayref_0.3.9": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"quickcheck\",\"req\":\"^1.0\"}],\"features\":{}}", "arrayvec_0.7.6": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"bencher\",\"req\":\"^0.1.4\"},{\"default_features\":false,\"name\":\"borsh\",\"optional\":true,\"req\":\"^1.2.0\"},{\"kind\":\"dev\",\"name\":\"matches\",\"req\":\"^0.1\"},{\"default_features\":false,\"name\":\"serde\",\"optional\":true,\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"serde_test\",\"req\":\"^1.0\"},{\"default_features\":false,\"name\":\"zeroize\",\"optional\":true,\"req\":\"^1.4\"}],\"features\":{\"default\":[\"std\"],\"std\":[]}}", + "ascii_1.1.0": "{\"dependencies\":[{\"name\":\"serde\",\"optional\":true,\"req\":\"^1.0.25\"},{\"name\":\"serde_test\",\"optional\":true,\"req\":\"^1.0\"}],\"features\":{\"alloc\":[],\"default\":[\"std\"],\"std\":[\"alloc\"]}}", "assert-json-diff_2.0.2": "{\"dependencies\":[{\"name\":\"serde\",\"req\":\"^1\"},{\"features\":[\"derive\"],\"kind\":\"dev\",\"name\":\"serde\",\"req\":\"^1\"},{\"name\":\"serde_json\",\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"version-sync\",\"req\":\"^0.8\"}],\"features\":{}}", "async-lock_3.4.1": "{\"dependencies\":[{\"default_features\":false,\"name\":\"event-listener\",\"req\":\"^5.0.0\"},{\"default_features\":false,\"name\":\"event-listener-strategy\",\"req\":\"^0.5.0\"},{\"kind\":\"dev\",\"name\":\"fastrand\",\"req\":\"^2.0.0\"},{\"kind\":\"dev\",\"name\":\"flume\",\"req\":\"^0.11.0\"},{\"kind\":\"dev\",\"name\":\"futures-lite\",\"req\":\"^2.0.0\"},{\"name\":\"loom\",\"optional\":true,\"req\":\"^0.7\",\"target\":\"cfg(loom)\"},{\"name\":\"pin-project-lite\",\"req\":\"^0.2.11\"},{\"kind\":\"dev\",\"name\":\"waker-fn\",\"req\":\"^1.1.0\"},{\"kind\":\"dev\",\"name\":\"wasm-bindgen-test\",\"req\":\"^0.3\",\"target\":\"cfg(target_family = \\\"wasm\\\")\"}],\"features\":{\"default\":[\"std\"],\"loom\":[\"event-listener/loom\",\"dep:loom\"],\"std\":[\"event-listener/std\",\"event-listener-strategy/std\"]}}", "async-stream-impl_0.3.6": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"futures-core\",\"req\":\"^0.3\"},{\"kind\":\"dev\",\"name\":\"futures-util\",\"req\":\"^0.3\"},{\"name\":\"proc-macro2\",\"req\":\"^1.0.60\"},{\"name\":\"quote\",\"req\":\"^1\"},{\"features\":[\"full\",\"visit-mut\"],\"name\":\"syn\",\"req\":\"^2.0.2\"},{\"features\":[\"full\"],\"kind\":\"dev\",\"name\":\"tokio\",\"req\":\"^1\"}],\"features\":{}}", @@ -798,6 +713,7 @@ "async-trait_0.1.89": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"futures\",\"req\":\"^0.3.30\"},{\"name\":\"proc-macro2\",\"req\":\"^1.0.74\"},{\"name\":\"quote\",\"req\":\"^1.0.35\"},{\"kind\":\"dev\",\"name\":\"rustversion\",\"req\":\"^1.0.13\"},{\"default_features\":false,\"features\":[\"clone-impls\",\"full\",\"parsing\",\"printing\",\"proc-macro\",\"visit-mut\"],\"name\":\"syn\",\"req\":\"^2.0.46\"},{\"kind\":\"dev\",\"name\":\"tracing\",\"req\":\"^0.1.40\"},{\"kind\":\"dev\",\"name\":\"tracing-attributes\",\"req\":\"^0.1.27\"},{\"features\":[\"diff\"],\"kind\":\"dev\",\"name\":\"trybuild\",\"req\":\"^1.0.81\"}],\"features\":{}}", "atomic-waker_1.1.2": "{\"dependencies\":[{\"default_features\":false,\"features\":[\"cargo_bench_support\"],\"kind\":\"dev\",\"name\":\"criterion\",\"req\":\"^0.4.0\"},{\"kind\":\"dev\",\"name\":\"futures\",\"req\":\"^0.3.5\"},{\"default_features\":false,\"name\":\"portable-atomic\",\"optional\":true,\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"rayon\",\"req\":\"^1.7.0\"}],\"features\":{}}", "atomic_0.6.1": "{\"dependencies\":[{\"name\":\"bytemuck\",\"req\":\"^1.13.1\"},{\"features\":[\"derive\"],\"kind\":\"dev\",\"name\":\"bytemuck\",\"req\":\"^1.13.1\"},{\"default_features\":false,\"name\":\"serde\",\"optional\":true,\"req\":\"^1.0.219\"},{\"default_features\":false,\"features\":[\"derive\"],\"kind\":\"dev\",\"name\":\"serde\",\"req\":\"^1.0.219\"},{\"kind\":\"dev\",\"name\":\"serde_json\",\"req\":\"^1.0.140\"}],\"features\":{\"default\":[\"fallback\"],\"fallback\":[],\"nightly\":[],\"serde\":[\"dep:serde\"],\"std\":[]}}", + "autocfg_1.5.0": "{\"dependencies\":[],\"features\":{}}", "autocfg_1.5.1": "{\"dependencies\":[],\"features\":{}}", "aws-config_1.8.14": "{\"dependencies\":[{\"features\":[\"test-util\"],\"name\":\"aws-credential-types\",\"req\":\"^1.2.12\"},{\"name\":\"aws-runtime\",\"req\":\"^1.7.0\"},{\"default_features\":false,\"name\":\"aws-sdk-signin\",\"optional\":true,\"req\":\"^1.5.0\"},{\"default_features\":false,\"name\":\"aws-sdk-sso\",\"optional\":true,\"req\":\"^1.94.0\"},{\"default_features\":false,\"name\":\"aws-sdk-ssooidc\",\"optional\":true,\"req\":\"^1.96.0\"},{\"default_features\":false,\"name\":\"aws-sdk-sts\",\"req\":\"^1.98.0\"},{\"name\":\"aws-smithy-async\",\"req\":\"^1.2.12\"},{\"features\":[\"rt-tokio\",\"test-util\"],\"kind\":\"dev\",\"name\":\"aws-smithy-async\",\"req\":\"^1.2.12\"},{\"name\":\"aws-smithy-http\",\"req\":\"^0.63.4\"},{\"features\":[\"default-client\",\"test-util\"],\"kind\":\"dev\",\"name\":\"aws-smithy-http-client\",\"req\":\"^1.1.10\"},{\"name\":\"aws-smithy-json\",\"req\":\"^0.62.4\"},{\"features\":[\"client\"],\"name\":\"aws-smithy-runtime\",\"req\":\"^1.10.1\"},{\"features\":[\"client\",\"test-util\"],\"kind\":\"dev\",\"name\":\"aws-smithy-runtime\",\"req\":\"^1.10.1\"},{\"features\":[\"client\"],\"name\":\"aws-smithy-runtime-api\",\"req\":\"^1.11.4\"},{\"features\":[\"test-util\"],\"kind\":\"dev\",\"name\":\"aws-smithy-runtime-api\",\"req\":\"^1.11.4\"},{\"name\":\"aws-smithy-types\",\"req\":\"^1.4.4\"},{\"name\":\"aws-types\",\"req\":\"^1.3.12\"},{\"name\":\"base64-simd\",\"optional\":true,\"req\":\"^0.8.0\"},{\"name\":\"bytes\",\"req\":\"^1.1.0\"},{\"name\":\"fastrand\",\"req\":\"^2.3.0\"},{\"default_features\":false,\"kind\":\"dev\",\"name\":\"futures-util\",\"req\":\"^0.3.29\"},{\"name\":\"hex\",\"optional\":true,\"req\":\"^0.4.3\"},{\"name\":\"http\",\"req\":\"^1\"},{\"name\":\"p256\",\"optional\":true,\"req\":\"^0.13.2\"},{\"default_features\":false,\"features\":[\"std\",\"std_rng\"],\"name\":\"rand\",\"optional\":true,\"req\":\"^0.8.5\"},{\"name\":\"ring\",\"optional\":true,\"req\":\"^0.17.5\"},{\"features\":[\"derive\"],\"kind\":\"dev\",\"name\":\"serde\",\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"serde_json\",\"req\":\"^1\"},{\"name\":\"sha2\",\"optional\":true,\"req\":\"^0.10.9\"},{\"features\":[\"parsing\"],\"name\":\"time\",\"req\":\"^0.3.4\"},{\"features\":[\"sync\"],\"name\":\"tokio\",\"req\":\"^1.13.1\"},{\"features\":[\"full\",\"test-util\"],\"kind\":\"dev\",\"name\":\"tokio\",\"req\":\"^1.23.1\"},{\"name\":\"tracing\",\"req\":\"^0.1\"},{\"features\":[\"fmt\",\"json\"],\"kind\":\"dev\",\"name\":\"tracing-subscriber\",\"req\":\"^0.3.16\"},{\"kind\":\"dev\",\"name\":\"tracing-test\",\"req\":\"^0.2.4\"},{\"name\":\"url\",\"req\":\"^2.5.4\"},{\"name\":\"uuid\",\"optional\":true,\"req\":\"^1.18.1\"},{\"name\":\"zeroize\",\"optional\":true,\"req\":\"^1\"}],\"features\":{\"allow-compilation\":[],\"behavior-version-latest\":[],\"client-hyper\":[\"aws-smithy-runtime/default-https-client\"],\"credentials-login\":[\"dep:aws-sdk-signin\",\"dep:sha2\",\"dep:zeroize\",\"dep:hex\",\"dep:base64-simd\",\"dep:uuid\",\"uuid?/v4\",\"dep:p256\",\"p256?/arithmetic\",\"p256?/pem\",\"dep:rand\"],\"credentials-process\":[\"tokio/process\"],\"default\":[\"default-https-client\",\"rt-tokio\",\"credentials-process\",\"sso\"],\"default-https-client\":[\"aws-smithy-runtime/default-https-client\"],\"rt-tokio\":[\"aws-smithy-async/rt-tokio\",\"aws-smithy-runtime/rt-tokio\",\"tokio/rt\"],\"rustls\":[\"client-hyper\"],\"sso\":[\"dep:aws-sdk-sso\",\"dep:aws-sdk-ssooidc\",\"dep:ring\",\"dep:hex\",\"dep:zeroize\",\"aws-smithy-runtime-api/http-auth\"],\"test-util\":[\"aws-runtime/test-util\"]}}", "aws-credential-types_1.2.12": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"async-trait\",\"req\":\"^0.1.74\"},{\"name\":\"aws-smithy-async\",\"req\":\"^1.2.12\"},{\"features\":[\"client\",\"http-auth\"],\"name\":\"aws-smithy-runtime-api\",\"req\":\"^1.11.4\"},{\"features\":[\"test-util\"],\"kind\":\"dev\",\"name\":\"aws-smithy-runtime-api\",\"req\":\"^1.11.4\"},{\"name\":\"aws-smithy-types\",\"req\":\"^1.4.4\"},{\"features\":[\"full\",\"test-util\",\"rt\"],\"kind\":\"dev\",\"name\":\"tokio\",\"req\":\"^1.23.1\"},{\"name\":\"zeroize\",\"req\":\"^1.7.0\"}],\"features\":{\"hardcoded-credentials\":[],\"test-util\":[\"aws-smithy-runtime-api/test-util\"]}}", @@ -843,8 +759,11 @@ "bitvec_1.0.1": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"bincode\",\"req\":\"^1.3\"},{\"kind\":\"dev\",\"name\":\"criterion\",\"req\":\"^0.3\"},{\"default_features\":false,\"name\":\"funty\",\"req\":\"^2.0\"},{\"name\":\"radium\",\"req\":\"^0.7\"},{\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.8\"},{\"default_features\":false,\"name\":\"serde\",\"optional\":true,\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"serde\",\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"serde_json\",\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"serde_test\",\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"static_assertions\",\"req\":\"^1\"},{\"name\":\"tap\",\"req\":\"^1\"},{\"default_features\":false,\"name\":\"wyz\",\"req\":\"^0.5\"}],\"features\":{\"alloc\":[],\"atomic\":[],\"default\":[\"atomic\",\"std\"],\"std\":[\"alloc\"],\"testing\":[]}}", "blake3_1.8.2": "{\"dependencies\":[{\"name\":\"arrayref\",\"req\":\"^0.3.5\"},{\"default_features\":false,\"name\":\"arrayvec\",\"req\":\"^0.7.4\"},{\"kind\":\"build\",\"name\":\"cc\",\"req\":\"^1.1.12\"},{\"name\":\"cfg-if\",\"req\":\"^1.0.0\"},{\"kind\":\"dev\",\"name\":\"ciborium\",\"req\":\"^0.2.2\"},{\"default_features\":false,\"name\":\"constant_time_eq\",\"req\":\"^0.3.1\"},{\"features\":[\"mac\"],\"name\":\"digest\",\"optional\":true,\"req\":\"^0.10.1\"},{\"kind\":\"dev\",\"name\":\"hex\",\"req\":\"^0.4.2\"},{\"kind\":\"dev\",\"name\":\"hmac\",\"req\":\"^0.12.0\"},{\"name\":\"memmap2\",\"optional\":true,\"req\":\"^0.9\"},{\"kind\":\"dev\",\"name\":\"page_size\",\"req\":\"^0.6.0\"},{\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.9.0\"},{\"kind\":\"dev\",\"name\":\"rand_chacha\",\"req\":\"^0.9.0\"},{\"name\":\"rayon-core\",\"optional\":true,\"req\":\"^1.12.1\"},{\"default_features\":false,\"features\":[\"derive\"],\"name\":\"serde\",\"optional\":true,\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"serde_json\",\"req\":\"^1.0.107\"},{\"kind\":\"dev\",\"name\":\"tempfile\",\"req\":\"^3.8.0\"},{\"default_features\":false,\"name\":\"zeroize\",\"optional\":true,\"req\":\"^1\"}],\"features\":{\"default\":[\"std\"],\"digest\":[\"dep:digest\"],\"mmap\":[\"std\",\"dep:memmap2\"],\"neon\":[],\"no_avx2\":[],\"no_avx512\":[],\"no_neon\":[],\"no_sse2\":[],\"no_sse41\":[],\"prefer_intrinsics\":[],\"pure\":[],\"rayon\":[\"dep:rayon-core\"],\"std\":[],\"traits-preview\":[\"dep:digest\"],\"wasm32_simd\":[],\"zeroize\":[\"dep:zeroize\",\"arrayvec/zeroize\"]}}", "block-buffer_0.10.4": "{\"dependencies\":[{\"name\":\"generic-array\",\"req\":\"^0.14\"}],\"features\":{}}", + "brotli-decompressor_4.0.3": "{\"dependencies\":[{\"name\":\"alloc-no-stdlib\",\"req\":\"~2.0\"},{\"name\":\"alloc-stdlib\",\"optional\":true,\"req\":\"~0.2\"}],\"features\":{\"benchmark\":[],\"default\":[\"std\"],\"disable-timer\":[],\"ffi-api\":[],\"pass-through-ffi-panics\":[],\"seccomp\":[],\"std\":[\"alloc-stdlib\"],\"unsafe\":[\"alloc-no-stdlib/unsafe\",\"alloc-stdlib/unsafe\"]}}", "bs58_0.5.1": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"assert_matches\",\"req\":\"^1.3.0\"},{\"kind\":\"dev\",\"name\":\"base58\",\"req\":\"^0.1.0\"},{\"kind\":\"dev\",\"name\":\"criterion\",\"req\":\"^0.5\"},{\"kind\":\"dev\",\"name\":\"rust-base58\",\"req\":\"^0.0.4\"},{\"default_features\":false,\"name\":\"sha2\",\"optional\":true,\"req\":\"^0.10\"},{\"name\":\"smallvec\",\"optional\":true,\"req\":\"^1\"},{\"default_features\":false,\"features\":[\"grab_spare_slice\"],\"name\":\"tinyvec\",\"optional\":true,\"req\":\"^1.6.0\"},{\"features\":[\"rustc_1_55\"],\"kind\":\"dev\",\"name\":\"tinyvec\",\"req\":\"^1.6.0\"}],\"features\":{\"alloc\":[\"tinyvec?/alloc\"],\"cb58\":[\"sha2\"],\"check\":[\"sha2\"],\"default\":[\"std\"],\"std\":[\"alloc\",\"tinyvec?/std\"]}}", "bson_2.15.0": "{\"dependencies\":[{\"name\":\"ahash\",\"req\":\"^0.8.0\"},{\"kind\":\"dev\",\"name\":\"assert_matches\",\"req\":\"^1.2\"},{\"name\":\"base64\",\"req\":\"^0.22.1\"},{\"name\":\"bitvec\",\"req\":\"^1.0.1\"},{\"default_features\":false,\"features\":[\"std\"],\"name\":\"chrono\",\"optional\":true,\"req\":\"^0.4.15\"},{\"default_features\":false,\"features\":[\"serde\",\"clock\",\"std\"],\"kind\":\"dev\",\"name\":\"chrono\",\"req\":\"^0.4\"},{\"kind\":\"dev\",\"name\":\"criterion\",\"req\":\"^0.3.0\"},{\"features\":[\"js\"],\"name\":\"getrandom\",\"req\":\"^0.2\",\"target\":\"cfg(all(target_arch = \\\"wasm32\\\", target_os = \\\"unknown\\\"))\"},{\"features\":[\"wasm_js\"],\"name\":\"getrandom_03\",\"package\":\"getrandom\",\"req\":\"^0.3\",\"target\":\"cfg(all(target_arch = \\\"wasm32\\\", target_os = \\\"unknown\\\"))\"},{\"name\":\"hex\",\"req\":\"^0.4.2\"},{\"name\":\"indexmap\",\"req\":\"^2.1.0\"},{\"name\":\"js-sys\",\"req\":\"^0.3\",\"target\":\"cfg(all(target_arch = \\\"wasm32\\\", target_os = \\\"unknown\\\"))\"},{\"name\":\"once_cell\",\"req\":\"^1.5.1\"},{\"kind\":\"dev\",\"name\":\"pretty_assertions\",\"req\":\"^0.6.1\"},{\"kind\":\"dev\",\"name\":\"proptest\",\"req\":\"^1.0.0\"},{\"name\":\"rand\",\"req\":\"^0.9\"},{\"features\":[\"derive\"],\"name\":\"serde\",\"req\":\"^1.0\"},{\"name\":\"serde_bytes\",\"req\":\"^0.11.5\"},{\"kind\":\"dev\",\"name\":\"serde_bytes\",\"req\":\"^0.11\"},{\"features\":[\"preserve_order\"],\"name\":\"serde_json\",\"req\":\"^1.0\"},{\"name\":\"serde_path_to_error\",\"optional\":true,\"req\":\"^0.1.16\"},{\"kind\":\"dev\",\"name\":\"serde_path_to_error\",\"req\":\"^0.1.16\"},{\"name\":\"serde_with\",\"optional\":true,\"req\":\"^1.3.1\"},{\"name\":\"serde_with-3\",\"optional\":true,\"package\":\"serde_with\",\"req\":\"^3.1.0\"},{\"features\":[\"formatting\",\"parsing\",\"macros\",\"large-dates\"],\"name\":\"time\",\"req\":\"^0.3.9\"},{\"features\":[\"serde\",\"v4\"],\"name\":\"uuid\",\"req\":\"^1.1.2\"},{\"features\":[\"serde\",\"v4\",\"js\"],\"name\":\"uuid\",\"req\":\"^1.1.2\",\"target\":\"cfg(all(target_arch = \\\"wasm32\\\", target_os = \\\"unknown\\\"))\"},{\"features\":[\"serde\",\"v4\"],\"name\":\"uuid-0_8\",\"optional\":true,\"package\":\"uuid\",\"req\":\"^0.8.1\"}],\"features\":{\"chrono-0_4\":[\"chrono\"],\"default\":[],\"hashable\":[],\"serde_path_to_error\":[\"dep:serde_path_to_error\"],\"time-0_3\":[],\"uncapped_max_size\":[],\"uuid-1\":[]}}", + "buf_redux_0.8.4": "{\"dependencies\":[{\"name\":\"memchr\",\"req\":\"^2.0\"},{\"name\":\"safemem\",\"req\":\"^0.3\"},{\"name\":\"slice-deque\",\"optional\":true,\"req\":\"^0.2\",\"target\":\"cfg(any(unix, windows))\"}],\"features\":{\"default\":[\"slice-deque\"],\"nightly\":[\"slice-deque/unstable\"]}}", + "bumpalo_3.20.2": "{\"dependencies\":[{\"default_features\":false,\"name\":\"allocator-api2\",\"optional\":true,\"req\":\"^0.2.8\"},{\"kind\":\"dev\",\"name\":\"blink-alloc\",\"req\":\"=0.4.0\"},{\"kind\":\"dev\",\"name\":\"criterion\",\"req\":\"^0.3.6\"},{\"kind\":\"dev\",\"name\":\"quickcheck\",\"req\":\"=1.0.3\"},{\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.8.5\"},{\"kind\":\"dev\",\"name\":\"rayon\",\"req\":\"=1.10.0\"},{\"kind\":\"dev\",\"name\":\"rayon-core\",\"req\":\"=1.12.1\"},{\"name\":\"serde\",\"optional\":true,\"req\":\"^1.0.171\"},{\"features\":[\"derive\"],\"kind\":\"dev\",\"name\":\"serde\",\"req\":\"^1.0.197\"},{\"kind\":\"dev\",\"name\":\"serde_json\",\"req\":\"^1.0.115\"}],\"features\":{\"allocator_api\":[],\"bench_allocator_api\":[\"allocator_api\",\"blink-alloc/nightly\"],\"boxed\":[],\"collections\":[],\"default\":[],\"serde\":[\"dep:serde\"],\"std\":[]}}", "bumpalo_3.20.3": "{\"dependencies\":[{\"default_features\":false,\"name\":\"allocator-api2\",\"optional\":true,\"req\":\"^0.2.8\"},{\"kind\":\"dev\",\"name\":\"blink-alloc\",\"req\":\"=0.4.0\"},{\"kind\":\"dev\",\"name\":\"criterion\",\"req\":\"^0.3.6\"},{\"kind\":\"dev\",\"name\":\"quickcheck\",\"req\":\"=1.0.3\"},{\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.8.5\"},{\"kind\":\"dev\",\"name\":\"rayon\",\"req\":\"=1.10.0\"},{\"kind\":\"dev\",\"name\":\"rayon-core\",\"req\":\"=1.12.1\"},{\"name\":\"serde\",\"optional\":true,\"req\":\"^1.0.171\"},{\"features\":[\"derive\"],\"kind\":\"dev\",\"name\":\"serde\",\"req\":\"^1.0.197\"},{\"kind\":\"dev\",\"name\":\"serde_json\",\"req\":\"^1.0.115\"}],\"features\":{\"allocator_api\":[],\"bench_allocator_api\":[\"allocator_api\",\"blink-alloc/nightly\"],\"boxed\":[],\"collections\":[],\"default\":[],\"serde\":[\"dep:serde\"],\"std\":[]}}", "byte-unit_5.1.6": "{\"dependencies\":[{\"name\":\"rocket\",\"optional\":true,\"req\":\"^0.5\"},{\"default_features\":false,\"name\":\"rust_decimal\",\"optional\":true,\"req\":\"^1\"},{\"default_features\":false,\"features\":[\"alloc\"],\"name\":\"serde\",\"optional\":true,\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"serde_json\",\"req\":\"^1\"},{\"name\":\"utf8-width\",\"req\":\"^0.1\"}],\"features\":{\"bit\":[\"rust_decimal\"],\"byte\":[\"rust_decimal\"],\"default\":[\"std\",\"byte\"],\"rocket\":[\"dep:rocket\",\"std\"],\"rust_decimal\":[\"dep:rust_decimal\"],\"serde\":[\"dep:serde\"],\"std\":[\"serde?/std\",\"rust_decimal?/std\"],\"u128\":[]}}", "bytemuck_1.25.0": "{\"dependencies\":[{\"name\":\"bytemuck_derive\",\"optional\":true,\"req\":\"^1.10.2\"},{\"name\":\"rustversion\",\"optional\":true,\"req\":\"^1.0.22\"}],\"features\":{\"aarch64_simd\":[],\"align_offset\":[],\"alloc_uninit\":[],\"avx512_simd\":[],\"const_zeroed\":[],\"derive\":[\"bytemuck_derive\"],\"extern_crate_alloc\":[],\"extern_crate_std\":[\"extern_crate_alloc\"],\"impl_core_error\":[],\"latest_stable_rust\":[\"aarch64_simd\",\"avx512_simd\",\"align_offset\",\"alloc_uninit\",\"const_zeroed\",\"derive\",\"impl_core_error\",\"min_const_generics\",\"must_cast\",\"must_cast_extra\",\"pod_saturating\",\"track_caller\",\"transparentwrapper_extra\",\"wasm_simd\",\"zeroable_atomics\",\"zeroable_maybe_uninit\",\"zeroable_unwind_fn\"],\"min_const_generics\":[],\"must_cast\":[],\"must_cast_extra\":[\"must_cast\"],\"nightly_docs\":[],\"nightly_float\":[],\"nightly_portable_simd\":[\"rustversion\"],\"nightly_stdsimd\":[],\"pod_saturating\":[],\"track_caller\":[],\"transparentwrapper_extra\":[],\"unsound_ptr_pod_impl\":[],\"wasm_simd\":[],\"zeroable_atomics\":[],\"zeroable_maybe_uninit\":[],\"zeroable_unwind_fn\":[]}}", @@ -853,20 +772,24 @@ "bytes_1.11.1": "{\"dependencies\":[{\"default_features\":false,\"features\":[\"require-cas\"],\"name\":\"extra-platforms\",\"optional\":true,\"package\":\"portable-atomic\",\"req\":\"^1.3\"},{\"kind\":\"dev\",\"name\":\"loom\",\"req\":\"^0.7\",\"target\":\"cfg(loom)\"},{\"default_features\":false,\"features\":[\"alloc\"],\"name\":\"serde\",\"optional\":true,\"req\":\"^1.0.60\"},{\"kind\":\"dev\",\"name\":\"serde_test\",\"req\":\"^1.0\"}],\"features\":{\"default\":[\"std\"],\"std\":[]}}", "camino_1.2.2": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"bincode\",\"req\":\"^1\"},{\"name\":\"proptest\",\"optional\":true,\"req\":\"^1.0.0\"},{\"features\":[\"derive\"],\"kind\":\"dev\",\"name\":\"serde\",\"req\":\"^1.0.223\"},{\"kind\":\"dev\",\"name\":\"serde_bytes\",\"req\":\"^0.11.8\"},{\"name\":\"serde_core\",\"optional\":true,\"req\":\"^1\"}],\"features\":{\"proptest1\":[\"dep:proptest\"],\"serde1\":[\"dep:serde_core\"]}}", "cbor-diag_0.1.12": "{\"dependencies\":[{\"default_features\":false,\"features\":[\"alloc\"],\"name\":\"bs58\",\"req\":\"^0.5.0\"},{\"default_features\":false,\"features\":[\"alloc\"],\"name\":\"chrono\",\"req\":\"^0.4.22\"},{\"default_features\":false,\"features\":[\"alloc\"],\"name\":\"data-encoding\",\"req\":\"^2.3.2\"},{\"default_features\":false,\"kind\":\"dev\",\"name\":\"data-encoding-macro\",\"req\":\"^0.1.12\"},{\"default_features\":false,\"name\":\"half\",\"req\":\"^2.1.0\"},{\"default_features\":false,\"kind\":\"dev\",\"name\":\"indoc\",\"req\":\"^2.0.1\"},{\"default_features\":false,\"features\":[\"std\"],\"name\":\"nom\",\"req\":\"^7.1.3\"},{\"default_features\":false,\"features\":[\"std\"],\"name\":\"num-bigint\",\"req\":\"^0.4.3\"},{\"default_features\":false,\"features\":[\"num-bigint\"],\"name\":\"num-rational\",\"req\":\"^0.4.1\"},{\"default_features\":false,\"name\":\"num-traits\",\"req\":\"^0.2.15\"},{\"default_features\":false,\"features\":[\"std\"],\"kind\":\"dev\",\"name\":\"pretty_assertions\",\"req\":\"^1.3.0\"},{\"default_features\":false,\"features\":[\"std\"],\"kind\":\"dev\",\"name\":\"proptest\",\"req\":\"^1.0.0\"},{\"default_features\":false,\"name\":\"separator\",\"req\":\"^0.4.1\"},{\"default_features\":false,\"name\":\"url\",\"req\":\"^2.3.1\"},{\"default_features\":false,\"name\":\"uuid\",\"req\":\"^1.1.2\"}],\"features\":{}}", + "cc_1.2.62": "{\"dependencies\":[{\"name\":\"find-msvc-tools\",\"req\":\"^0.1.9\"},{\"default_features\":false,\"name\":\"jobserver\",\"optional\":true,\"req\":\"^0.1.30\"},{\"default_features\":false,\"name\":\"libc\",\"optional\":true,\"req\":\"^0.2.62\",\"target\":\"cfg(unix)\"},{\"name\":\"shlex\",\"req\":\"^1.3.0\"},{\"kind\":\"dev\",\"name\":\"tempfile\",\"req\":\"^3\"}],\"features\":{\"jobserver\":[],\"parallel\":[\"dep:libc\",\"dep:jobserver\"]}}", "cc_1.2.63": "{\"dependencies\":[{\"name\":\"find-msvc-tools\",\"req\":\"^0.1.9\"},{\"default_features\":false,\"name\":\"jobserver\",\"optional\":true,\"req\":\"^0.1.30\"},{\"default_features\":false,\"name\":\"libc\",\"optional\":true,\"req\":\"^0.2.62\",\"target\":\"cfg(unix)\"},{\"name\":\"shlex\",\"req\":\"^2.0.1\"},{\"kind\":\"dev\",\"name\":\"tempfile\",\"req\":\"^3\"}],\"features\":{\"jobserver\":[],\"parallel\":[\"dep:libc\",\"dep:jobserver\"]}}", "cesu8_1.1.0": "{\"dependencies\":[],\"features\":{\"unstable\":[]}}", "cfg-if_1.0.4": "{\"dependencies\":[{\"name\":\"core\",\"optional\":true,\"package\":\"rustc-std-workspace-core\",\"req\":\"^1.0.0\"}],\"features\":{\"rustc-dep-of-std\":[\"core\"]}}", "chacha20_0.10.0": "{\"dependencies\":[{\"name\":\"cfg-if\",\"req\":\"^1\"},{\"features\":[\"stream-wrapper\"],\"name\":\"cipher\",\"optional\":true,\"req\":\"^0.5\"},{\"features\":[\"dev\"],\"kind\":\"dev\",\"name\":\"cipher\",\"req\":\"^0.5\"},{\"name\":\"cpufeatures\",\"req\":\"^0.3\",\"target\":\"cfg(any(target_arch = \\\"x86_64\\\", target_arch = \\\"x86\\\"))\"},{\"kind\":\"dev\",\"name\":\"hex-literal\",\"req\":\"^1\"},{\"default_features\":false,\"name\":\"rand_core\",\"optional\":true,\"req\":\"^0.10\"},{\"default_features\":false,\"name\":\"zeroize\",\"optional\":true,\"req\":\"^1.8.1\"}],\"features\":{\"default\":[\"cipher\"],\"legacy\":[\"cipher\"],\"rng\":[\"dep:rand_core\"],\"xchacha\":[\"cipher\"]}}", "chrono_0.4.44": "{\"dependencies\":[{\"features\":[\"derive\"],\"name\":\"arbitrary\",\"optional\":true,\"req\":\"^1.0.0\"},{\"kind\":\"dev\",\"name\":\"bincode\",\"req\":\"^1.3.0\"},{\"name\":\"defmt\",\"optional\":true,\"req\":\"^1.0.1\"},{\"features\":[\"fallback\"],\"name\":\"iana-time-zone\",\"optional\":true,\"req\":\"^0.1.45\",\"target\":\"cfg(unix)\"},{\"name\":\"js-sys\",\"optional\":true,\"req\":\"^0.3\",\"target\":\"cfg(all(target_arch = \\\"wasm32\\\", not(any(target_os = \\\"emscripten\\\", target_os = \\\"wasi\\\"))))\"},{\"default_features\":false,\"name\":\"num-traits\",\"req\":\"^0.2\"},{\"name\":\"pure-rust-locales\",\"optional\":true,\"req\":\"^0.8.2\"},{\"default_features\":false,\"name\":\"rkyv\",\"optional\":true,\"req\":\"^0.7.43\"},{\"default_features\":false,\"name\":\"serde\",\"optional\":true,\"req\":\"^1.0.99\"},{\"default_features\":false,\"kind\":\"dev\",\"name\":\"serde_derive\",\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"serde_json\",\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"similar-asserts\",\"req\":\"^1.6.1\"},{\"name\":\"wasm-bindgen\",\"optional\":true,\"req\":\"^0.2\",\"target\":\"cfg(all(target_arch = \\\"wasm32\\\", not(any(target_os = \\\"emscripten\\\", target_os = \\\"wasi\\\"))))\"},{\"kind\":\"dev\",\"name\":\"wasm-bindgen-test\",\"req\":\"^0.3\",\"target\":\"cfg(all(target_arch = \\\"wasm32\\\", not(any(target_os = \\\"emscripten\\\", target_os = \\\"wasi\\\"))))\"},{\"kind\":\"dev\",\"name\":\"windows-bindgen\",\"req\":\"^0.66\"},{\"name\":\"windows-link\",\"optional\":true,\"req\":\"^0.2\",\"target\":\"cfg(windows)\"}],\"features\":{\"__internal_bench\":[],\"alloc\":[],\"clock\":[\"winapi\",\"iana-time-zone\",\"now\"],\"core-error\":[],\"default\":[\"clock\",\"std\",\"oldtime\",\"wasmbind\"],\"defmt\":[\"dep:defmt\",\"pure-rust-locales?/defmt\"],\"libc\":[],\"now\":[\"std\"],\"oldtime\":[],\"rkyv\":[\"dep:rkyv\",\"rkyv/size_32\"],\"rkyv-16\":[\"dep:rkyv\",\"rkyv?/size_16\"],\"rkyv-32\":[\"dep:rkyv\",\"rkyv?/size_32\"],\"rkyv-64\":[\"dep:rkyv\",\"rkyv?/size_64\"],\"rkyv-validation\":[\"rkyv?/validation\"],\"std\":[\"alloc\"],\"unstable-locales\":[\"pure-rust-locales\"],\"wasmbind\":[\"wasm-bindgen\",\"js-sys\"],\"winapi\":[\"windows-link\"]}}", + "chunked_transfer_1.5.0": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"criterion\",\"req\":\"^0.3\"}],\"features\":{}}", "ciborium-io_0.2.2": "{\"dependencies\":[],\"features\":{\"alloc\":[],\"std\":[\"alloc\"]}}", "ciborium-ll_0.2.2": "{\"dependencies\":[{\"name\":\"ciborium-io\",\"req\":\"^0.2.2\"},{\"default_features\":false,\"name\":\"half\",\"req\":\"^2.2\"},{\"kind\":\"dev\",\"name\":\"hex\",\"req\":\"^0.4\"}],\"features\":{\"alloc\":[],\"std\":[\"alloc\",\"half/std\"]}}", "ciborium_0.2.2": "{\"dependencies\":[{\"features\":[\"alloc\"],\"name\":\"ciborium-io\",\"req\":\"^0.2.2\"},{\"name\":\"ciborium-ll\",\"req\":\"^0.2.2\"},{\"kind\":\"dev\",\"name\":\"hex\",\"req\":\"^0.4\"},{\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.8\"},{\"kind\":\"dev\",\"name\":\"rstest\",\"req\":\"^0.11\"},{\"default_features\":false,\"features\":[\"alloc\",\"derive\"],\"name\":\"serde\",\"req\":\"^1.0.100\"},{\"kind\":\"dev\",\"name\":\"serde_bytes\",\"req\":\"^0.11\"}],\"features\":{\"default\":[\"std\"],\"std\":[\"ciborium-io/std\",\"serde/std\"]}}", "clap_4.5.50": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"automod\",\"req\":\"^1.0.14\"},{\"default_features\":false,\"kind\":\"dev\",\"name\":\"clap-cargo\",\"req\":\"^0.15.0\"},{\"default_features\":false,\"name\":\"clap_builder\",\"req\":\"=4.5.50\"},{\"name\":\"clap_derive\",\"optional\":true,\"req\":\"=4.5.49\"},{\"kind\":\"dev\",\"name\":\"jiff\",\"req\":\"^0.2.3\"},{\"kind\":\"dev\",\"name\":\"rustversion\",\"req\":\"^1.0.15\"},{\"kind\":\"dev\",\"name\":\"semver\",\"req\":\"^1.0.26\"},{\"kind\":\"dev\",\"name\":\"shlex\",\"req\":\"^1.3.0\"},{\"features\":[\"term-svg\"],\"kind\":\"dev\",\"name\":\"snapbox\",\"req\":\"^0.6.16\"},{\"kind\":\"dev\",\"name\":\"trybuild\",\"req\":\"^1.0.91\"},{\"default_features\":false,\"features\":[\"color-auto\",\"diff\",\"examples\"],\"kind\":\"dev\",\"name\":\"trycmd\",\"req\":\"^0.15.3\"}],\"features\":{\"cargo\":[\"clap_builder/cargo\"],\"color\":[\"clap_builder/color\"],\"debug\":[\"clap_builder/debug\",\"clap_derive?/debug\"],\"default\":[\"std\",\"color\",\"help\",\"usage\",\"error-context\",\"suggestions\"],\"deprecated\":[\"clap_builder/deprecated\",\"clap_derive?/deprecated\"],\"derive\":[\"dep:clap_derive\"],\"env\":[\"clap_builder/env\"],\"error-context\":[\"clap_builder/error-context\"],\"help\":[\"clap_builder/help\"],\"std\":[\"clap_builder/std\"],\"string\":[\"clap_builder/string\"],\"suggestions\":[\"clap_builder/suggestions\"],\"unicode\":[\"clap_builder/unicode\"],\"unstable-derive-ui-tests\":[],\"unstable-doc\":[\"clap_builder/unstable-doc\",\"derive\"],\"unstable-ext\":[\"clap_builder/unstable-ext\"],\"unstable-markdown\":[\"clap_derive/unstable-markdown\"],\"unstable-styles\":[\"clap_builder/unstable-styles\"],\"unstable-v5\":[\"clap_builder/unstable-v5\",\"clap_derive?/unstable-v5\",\"deprecated\"],\"usage\":[\"clap_builder/usage\"],\"wrap_help\":[\"clap_builder/wrap_help\"]}}", "clap_4.6.0": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"automod\",\"req\":\"^1.0.16\"},{\"default_features\":false,\"kind\":\"dev\",\"name\":\"clap-cargo\",\"req\":\"^0.15.2\"},{\"default_features\":false,\"name\":\"clap_builder\",\"req\":\"=4.6.0\"},{\"name\":\"clap_derive\",\"optional\":true,\"req\":\"=4.6.0\"},{\"kind\":\"dev\",\"name\":\"jiff\",\"req\":\"^0.2.23\"},{\"kind\":\"dev\",\"name\":\"rustversion\",\"req\":\"^1.0.22\"},{\"kind\":\"dev\",\"name\":\"semver\",\"req\":\"^1.0.27\"},{\"kind\":\"dev\",\"name\":\"shlex\",\"req\":\"^1.3.0\"},{\"features\":[\"term-svg\"],\"kind\":\"dev\",\"name\":\"snapbox\",\"req\":\"^1.1.0\"},{\"kind\":\"dev\",\"name\":\"trybuild\",\"req\":\"^1.0.116\"},{\"default_features\":false,\"features\":[\"color-auto\",\"diff\",\"examples\"],\"kind\":\"dev\",\"name\":\"trycmd\",\"req\":\"^1.1.1\"}],\"features\":{\"cargo\":[\"clap_builder/cargo\"],\"color\":[\"clap_builder/color\"],\"debug\":[\"clap_builder/debug\",\"clap_derive?/debug\"],\"default\":[\"std\",\"color\",\"help\",\"usage\",\"error-context\",\"suggestions\"],\"deprecated\":[\"clap_builder/deprecated\",\"clap_derive?/deprecated\"],\"derive\":[\"dep:clap_derive\"],\"env\":[\"clap_builder/env\"],\"error-context\":[\"clap_builder/error-context\"],\"help\":[\"clap_builder/help\"],\"std\":[\"clap_builder/std\"],\"string\":[\"clap_builder/string\"],\"suggestions\":[\"clap_builder/suggestions\"],\"unicode\":[\"clap_builder/unicode\"],\"unstable-derive-ui-tests\":[],\"unstable-doc\":[\"clap_builder/unstable-doc\",\"derive\"],\"unstable-ext\":[\"clap_builder/unstable-ext\"],\"unstable-markdown\":[\"clap_derive/unstable-markdown\"],\"unstable-styles\":[\"clap_builder/unstable-styles\"],\"unstable-v5\":[\"clap_builder/unstable-v5\",\"clap_derive?/unstable-v5\",\"deprecated\"],\"usage\":[\"clap_builder/usage\"],\"wrap_help\":[\"clap_builder/wrap_help\"]}}", + "clap_4.6.1": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"automod\",\"req\":\"^1.0.16\"},{\"default_features\":false,\"kind\":\"dev\",\"name\":\"clap-cargo\",\"req\":\"^0.15.2\"},{\"default_features\":false,\"name\":\"clap_builder\",\"req\":\"=4.6.0\"},{\"name\":\"clap_derive\",\"optional\":true,\"req\":\"=4.6.1\"},{\"kind\":\"dev\",\"name\":\"jiff\",\"req\":\"^0.2.23\"},{\"kind\":\"dev\",\"name\":\"rustversion\",\"req\":\"^1.0.22\"},{\"kind\":\"dev\",\"name\":\"semver\",\"req\":\"^1.0.27\"},{\"kind\":\"dev\",\"name\":\"shlex\",\"req\":\"^1.3.0\"},{\"features\":[\"term-svg\"],\"kind\":\"dev\",\"name\":\"snapbox\",\"req\":\"^1.2.0\"},{\"kind\":\"dev\",\"name\":\"trybuild\",\"req\":\"^1.0.116\"},{\"default_features\":false,\"features\":[\"color-auto\",\"diff\",\"examples\"],\"kind\":\"dev\",\"name\":\"trycmd\",\"req\":\"^1.2.0\"}],\"features\":{\"cargo\":[\"clap_builder/cargo\"],\"color\":[\"clap_builder/color\"],\"debug\":[\"clap_builder/debug\",\"clap_derive?/debug\"],\"default\":[\"std\",\"color\",\"help\",\"usage\",\"error-context\",\"suggestions\"],\"deprecated\":[\"clap_builder/deprecated\",\"clap_derive?/deprecated\"],\"derive\":[\"dep:clap_derive\"],\"env\":[\"clap_builder/env\"],\"error-context\":[\"clap_builder/error-context\"],\"help\":[\"clap_builder/help\"],\"std\":[\"clap_builder/std\"],\"string\":[\"clap_builder/string\"],\"suggestions\":[\"clap_builder/suggestions\"],\"unicode\":[\"clap_builder/unicode\"],\"unstable-derive-ui-tests\":[],\"unstable-doc\":[\"clap_builder/unstable-doc\",\"derive\"],\"unstable-ext\":[\"clap_builder/unstable-ext\"],\"unstable-markdown\":[\"clap_derive/unstable-markdown\"],\"unstable-styles\":[\"clap_builder/unstable-styles\"],\"unstable-v5\":[\"clap_builder/unstable-v5\",\"clap_derive?/unstable-v5\",\"deprecated\"],\"usage\":[\"clap_builder/usage\"],\"wrap_help\":[\"clap_builder/wrap_help\"]}}", "clap_builder_4.5.50": "{\"dependencies\":[{\"name\":\"anstream\",\"optional\":true,\"req\":\"^0.6.7\"},{\"name\":\"anstyle\",\"req\":\"^1.0.8\"},{\"name\":\"backtrace\",\"optional\":true,\"req\":\"^0.3.73\"},{\"name\":\"clap_lex\",\"req\":\"^0.7.4\"},{\"kind\":\"dev\",\"name\":\"color-print\",\"req\":\"^0.3.6\"},{\"kind\":\"dev\",\"name\":\"snapbox\",\"req\":\"^0.6.16\"},{\"kind\":\"dev\",\"name\":\"static_assertions\",\"req\":\"^1.1.0\"},{\"name\":\"strsim\",\"optional\":true,\"req\":\"^0.11.0\"},{\"name\":\"terminal_size\",\"optional\":true,\"req\":\"^0.4.0\"},{\"kind\":\"dev\",\"name\":\"unic-emoji-char\",\"req\":\"^0.9.0\"},{\"name\":\"unicase\",\"optional\":true,\"req\":\"^2.6.0\"},{\"name\":\"unicode-width\",\"optional\":true,\"req\":\"^0.2.0\"}],\"features\":{\"cargo\":[],\"color\":[\"dep:anstream\"],\"debug\":[\"dep:backtrace\"],\"default\":[\"std\",\"color\",\"help\",\"usage\",\"error-context\",\"suggestions\"],\"deprecated\":[],\"env\":[],\"error-context\":[],\"help\":[],\"std\":[\"anstyle/std\"],\"string\":[],\"suggestions\":[\"dep:strsim\",\"error-context\"],\"unicode\":[\"dep:unicode-width\",\"dep:unicase\"],\"unstable-doc\":[\"cargo\",\"wrap_help\",\"env\",\"unicode\",\"string\",\"unstable-ext\"],\"unstable-ext\":[],\"unstable-styles\":[\"color\"],\"unstable-v5\":[\"deprecated\"],\"usage\":[],\"wrap_help\":[\"help\",\"dep:terminal_size\"]}}", "clap_builder_4.6.0": "{\"dependencies\":[{\"name\":\"anstream\",\"optional\":true,\"req\":\"^1.0.0\"},{\"name\":\"anstyle\",\"req\":\"^1.0.13\"},{\"name\":\"backtrace\",\"optional\":true,\"req\":\"^0.3.76\"},{\"name\":\"clap_lex\",\"req\":\"^1.0.0\"},{\"kind\":\"dev\",\"name\":\"color-print\",\"req\":\"^0.3.7\"},{\"kind\":\"dev\",\"name\":\"snapbox\",\"req\":\"^1.1.0\"},{\"kind\":\"dev\",\"name\":\"static_assertions\",\"req\":\"^1.1.0\"},{\"name\":\"strsim\",\"optional\":true,\"req\":\"^0.11.1\"},{\"name\":\"terminal_size\",\"optional\":true,\"req\":\"^0.4.3\"},{\"kind\":\"dev\",\"name\":\"unic-emoji-char\",\"req\":\"^0.9.0\"},{\"name\":\"unicase\",\"optional\":true,\"req\":\"^2.9.0\"},{\"name\":\"unicode-width\",\"optional\":true,\"req\":\"^0.2.2\"}],\"features\":{\"cargo\":[],\"color\":[\"dep:anstream\"],\"debug\":[\"dep:backtrace\"],\"default\":[\"std\",\"color\",\"help\",\"usage\",\"error-context\",\"suggestions\"],\"deprecated\":[],\"env\":[],\"error-context\":[],\"help\":[],\"std\":[\"anstyle/std\"],\"string\":[],\"suggestions\":[\"dep:strsim\",\"error-context\"],\"unicode\":[\"dep:unicode-width\",\"dep:unicase\"],\"unstable-doc\":[\"cargo\",\"wrap_help\",\"env\",\"unicode\",\"string\",\"unstable-ext\"],\"unstable-ext\":[],\"unstable-styles\":[\"color\"],\"unstable-v5\":[\"deprecated\"],\"usage\":[],\"wrap_help\":[\"help\",\"dep:terminal_size\"]}}", "clap_derive_4.5.49": "{\"dependencies\":[{\"name\":\"anstyle\",\"optional\":true,\"req\":\"^1.0.10\"},{\"name\":\"heck\",\"req\":\"^0.5.0\"},{\"name\":\"proc-macro2\",\"req\":\"^1.0.69\"},{\"default_features\":false,\"name\":\"pulldown-cmark\",\"optional\":true,\"req\":\"^0.13.0\"},{\"name\":\"quote\",\"req\":\"^1.0.9\"},{\"features\":[\"full\"],\"name\":\"syn\",\"req\":\"^2.0.8\"}],\"features\":{\"debug\":[],\"default\":[],\"deprecated\":[],\"raw-deprecated\":[\"deprecated\"],\"unstable-markdown\":[\"dep:pulldown-cmark\",\"dep:anstyle\"],\"unstable-v5\":[\"deprecated\"]}}", "clap_derive_4.6.0": "{\"dependencies\":[{\"name\":\"anstyle\",\"optional\":true,\"req\":\"^1.0.13\"},{\"name\":\"heck\",\"req\":\"^0.5.0\"},{\"name\":\"proc-macro2\",\"req\":\"^1.0.106\"},{\"default_features\":false,\"name\":\"pulldown-cmark\",\"optional\":true,\"req\":\"^0.13.1\"},{\"name\":\"quote\",\"req\":\"^1.0.45\"},{\"features\":[\"full\"],\"name\":\"syn\",\"req\":\"^2.0.117\"}],\"features\":{\"debug\":[],\"default\":[],\"deprecated\":[],\"raw-deprecated\":[\"deprecated\"],\"unstable-markdown\":[\"dep:pulldown-cmark\",\"dep:anstyle\"],\"unstable-v5\":[\"deprecated\"]}}", + "clap_derive_4.6.1": "{\"dependencies\":[{\"name\":\"anstyle\",\"optional\":true,\"req\":\"^1.0.14\"},{\"name\":\"heck\",\"req\":\"^0.5.0\"},{\"name\":\"proc-macro2\",\"req\":\"^1.0.106\"},{\"default_features\":false,\"name\":\"pulldown-cmark\",\"optional\":true,\"req\":\"^0.13.3\"},{\"name\":\"quote\",\"req\":\"^1.0.45\"},{\"features\":[\"full\"],\"name\":\"syn\",\"req\":\"^2.0.117\"}],\"features\":{\"debug\":[],\"default\":[],\"deprecated\":[],\"raw-deprecated\":[\"deprecated\"],\"unstable-markdown\":[\"dep:pulldown-cmark\",\"dep:anstyle\"],\"unstable-v5\":[\"deprecated\"]}}", "clap_lex_0.7.6": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"automod\",\"req\":\"^1.0.14\"}],\"features\":{}}", "clap_lex_1.1.0": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"automod\",\"req\":\"^1.0.16\"}],\"features\":{}}", "colorchoice_1.0.4": "{\"dependencies\":[],\"features\":{}}", @@ -893,6 +816,8 @@ "crc_3.3.0": "{\"dependencies\":[{\"name\":\"crc-catalog\",\"req\":\"^2.4.0\"}],\"features\":{}}", "critical-section_1.2.0": "{\"dependencies\":[],\"features\":{\"restore-state-bool\":[],\"restore-state-none\":[],\"restore-state-u16\":[],\"restore-state-u32\":[],\"restore-state-u64\":[],\"restore-state-u8\":[],\"restore-state-usize\":[],\"std\":[\"restore-state-bool\"]}}", "crossbeam-channel_0.5.15": "{\"dependencies\":[{\"default_features\":false,\"name\":\"crossbeam-utils\",\"req\":\"^0.8.18\"},{\"kind\":\"dev\",\"name\":\"num_cpus\",\"req\":\"^1.13.0\"},{\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.8\"},{\"kind\":\"dev\",\"name\":\"signal-hook\",\"req\":\"^0.3\"}],\"features\":{\"default\":[\"std\"],\"std\":[\"crossbeam-utils/std\"]}}", + "crossbeam-deque_0.8.6": "{\"dependencies\":[{\"default_features\":false,\"name\":\"crossbeam-epoch\",\"req\":\"^0.9.17\"},{\"default_features\":false,\"name\":\"crossbeam-utils\",\"req\":\"^0.8.18\"},{\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.8\"}],\"features\":{\"default\":[\"std\"],\"std\":[\"crossbeam-epoch/std\",\"crossbeam-utils/std\"]}}", + "crossbeam-epoch_0.9.18": "{\"dependencies\":[{\"default_features\":false,\"name\":\"crossbeam-utils\",\"req\":\"^0.8.18\"},{\"name\":\"loom-crate\",\"optional\":true,\"package\":\"loom\",\"req\":\"^0.7.1\",\"target\":\"cfg(crossbeam_loom)\"},{\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.8\"}],\"features\":{\"alloc\":[],\"default\":[\"std\"],\"loom\":[\"loom-crate\",\"crossbeam-utils/loom\"],\"nightly\":[\"crossbeam-utils/nightly\"],\"std\":[\"alloc\",\"crossbeam-utils/std\"]}}", "crossbeam-epoch_0.9.20": "{\"dependencies\":[{\"default_features\":false,\"name\":\"crossbeam-utils\",\"req\":\"^0.8.18\"},{\"name\":\"loom-crate\",\"optional\":true,\"package\":\"loom\",\"req\":\"^0.7.1\",\"target\":\"cfg(crossbeam_loom)\"},{\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.8\"}],\"features\":{\"alloc\":[],\"default\":[\"std\"],\"loom\":[\"loom-crate\",\"crossbeam-utils/loom\"],\"nightly\":[\"crossbeam-utils/nightly\"],\"std\":[\"alloc\",\"crossbeam-utils/std\"]}}", "crossbeam-utils_0.8.21": "{\"dependencies\":[{\"name\":\"loom\",\"optional\":true,\"req\":\"^0.7.1\",\"target\":\"cfg(crossbeam_loom)\"},{\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.8\"}],\"features\":{\"default\":[\"std\"],\"nightly\":[],\"std\":[]}}", "crunchy_0.2.4": "{\"dependencies\":[],\"features\":{\"default\":[\"limit_128\"],\"limit_1024\":[],\"limit_128\":[],\"limit_2048\":[],\"limit_256\":[],\"limit_512\":[],\"limit_64\":[],\"std\":[]}}", @@ -900,9 +825,9 @@ "crypto-common_0.1.6": "{\"dependencies\":[{\"features\":[\"more_lengths\"],\"name\":\"generic-array\",\"req\":\"^0.14.4\"},{\"name\":\"rand_core\",\"optional\":true,\"req\":\"^0.6\"},{\"name\":\"typenum\",\"req\":\"^1.14\"}],\"features\":{\"getrandom\":[\"rand_core/getrandom\"],\"std\":[]}}", "curve25519-dalek-derive_0.1.1": "{\"dependencies\":[{\"name\":\"proc-macro2\",\"req\":\"^1.0.66\"},{\"name\":\"quote\",\"req\":\"^1.0.31\"},{\"features\":[\"full\"],\"name\":\"syn\",\"req\":\"^2.0.27\"}],\"features\":{}}", "curve25519-dalek_4.1.3": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"bincode\",\"req\":\"^1\"},{\"name\":\"cfg-if\",\"req\":\"^1\"},{\"name\":\"cpufeatures\",\"req\":\"^0.2.6\",\"target\":\"cfg(target_arch = \\\"x86_64\\\")\"},{\"features\":[\"html_reports\"],\"kind\":\"dev\",\"name\":\"criterion\",\"req\":\"^0.5\"},{\"name\":\"curve25519-dalek-derive\",\"req\":\"^0.1\",\"target\":\"cfg(all(not(curve25519_dalek_backend = \\\"fiat\\\"), not(curve25519_dalek_backend = \\\"serial\\\"), target_arch = \\\"x86_64\\\"))\"},{\"default_features\":false,\"name\":\"digest\",\"optional\":true,\"req\":\"^0.10\"},{\"default_features\":false,\"name\":\"ff\",\"optional\":true,\"req\":\"^0.13\"},{\"default_features\":false,\"name\":\"fiat-crypto\",\"req\":\"^0.2.1\",\"target\":\"cfg(curve25519_dalek_backend = \\\"fiat\\\")\"},{\"default_features\":false,\"name\":\"group\",\"optional\":true,\"req\":\"^0.13\"},{\"kind\":\"dev\",\"name\":\"hex\",\"req\":\"^0.4.2\"},{\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.8\"},{\"default_features\":false,\"name\":\"rand_core\",\"optional\":true,\"req\":\"^0.6.4\"},{\"default_features\":false,\"features\":[\"getrandom\"],\"kind\":\"dev\",\"name\":\"rand_core\",\"req\":\"^0.6\"},{\"kind\":\"build\",\"name\":\"rustc_version\",\"req\":\"^0.4.0\"},{\"default_features\":false,\"features\":[\"derive\"],\"name\":\"serde\",\"optional\":true,\"req\":\"^1.0\"},{\"default_features\":false,\"kind\":\"dev\",\"name\":\"sha2\",\"req\":\"^0.10\"},{\"default_features\":false,\"name\":\"subtle\",\"req\":\"^2.3.0\"},{\"default_features\":false,\"name\":\"zeroize\",\"optional\":true,\"req\":\"^1\"}],\"features\":{\"alloc\":[\"zeroize?/alloc\"],\"default\":[\"alloc\",\"precomputed-tables\",\"zeroize\"],\"group\":[\"dep:group\",\"rand_core\"],\"group-bits\":[\"group\",\"ff/bits\"],\"legacy_compatibility\":[],\"precomputed-tables\":[]}}", - "darling_0.21.3": "{\"dependencies\":[{\"name\":\"darling_core\",\"req\":\"=0.21.3\"},{\"name\":\"darling_macro\",\"req\":\"=0.21.3\"},{\"kind\":\"dev\",\"name\":\"proc-macro2\",\"req\":\"^1.0.86\"},{\"kind\":\"dev\",\"name\":\"quote\",\"req\":\"^1.0.18\"},{\"kind\":\"dev\",\"name\":\"rustversion\",\"req\":\"^1.0.9\",\"target\":\"cfg(compiletests)\"},{\"kind\":\"dev\",\"name\":\"syn\",\"req\":\"^2.0.15\"},{\"kind\":\"dev\",\"name\":\"trybuild\",\"req\":\"^1.0.89\",\"target\":\"cfg(compiletests)\"}],\"features\":{\"default\":[\"suggestions\"],\"diagnostics\":[\"darling_core/diagnostics\"],\"serde\":[\"darling_core/serde\"],\"suggestions\":[\"darling_core/suggestions\"]}}", - "darling_core_0.21.3": "{\"dependencies\":[{\"name\":\"fnv\",\"req\":\"^1.0.7\"},{\"name\":\"ident_case\",\"req\":\"^1.0.1\"},{\"name\":\"proc-macro2\",\"req\":\"^1.0.86\"},{\"name\":\"quote\",\"req\":\"^1.0.18\"},{\"name\":\"serde\",\"optional\":true,\"req\":\"^1.0.210\"},{\"kind\":\"dev\",\"name\":\"serde_json\",\"req\":\"^1.0.140\"},{\"name\":\"strsim\",\"optional\":true,\"req\":\"^0.11.1\"},{\"features\":[\"full\",\"extra-traits\"],\"name\":\"syn\",\"req\":\"^2.0.15\"}],\"features\":{\"diagnostics\":[],\"suggestions\":[\"strsim\"]}}", - "darling_macro_0.21.3": "{\"dependencies\":[{\"name\":\"darling_core\",\"req\":\"=0.21.3\"},{\"name\":\"quote\",\"req\":\"^1.0.18\"},{\"name\":\"syn\",\"req\":\"^2.0.15\"}],\"features\":{}}", + "darling_0.23.0": "{\"dependencies\":[{\"name\":\"darling_core\",\"req\":\"=0.23.0\"},{\"name\":\"darling_macro\",\"req\":\"=0.23.0\"},{\"kind\":\"dev\",\"name\":\"proc-macro2\",\"req\":\"^1.0.86\"},{\"kind\":\"dev\",\"name\":\"quote\",\"req\":\"^1.0.18\"},{\"kind\":\"dev\",\"name\":\"rustversion\",\"req\":\"^1.0.9\",\"target\":\"cfg(compiletests)\"},{\"kind\":\"dev\",\"name\":\"syn\",\"req\":\"^2.0.15\"},{\"kind\":\"dev\",\"name\":\"trybuild\",\"req\":\"^1.0.89\",\"target\":\"cfg(compiletests)\"}],\"features\":{\"default\":[\"suggestions\"],\"diagnostics\":[\"darling_core/diagnostics\"],\"serde\":[\"darling_core/serde\"],\"suggestions\":[\"darling_core/suggestions\"]}}", + "darling_core_0.23.0": "{\"dependencies\":[{\"name\":\"ident_case\",\"req\":\"^1.0.1\"},{\"name\":\"proc-macro2\",\"req\":\"^1.0.86\"},{\"name\":\"quote\",\"req\":\"^1.0.18\"},{\"name\":\"serde\",\"optional\":true,\"req\":\"^1.0.210\"},{\"kind\":\"dev\",\"name\":\"serde_json\",\"req\":\"^1.0.140\"},{\"name\":\"strsim\",\"optional\":true,\"req\":\"^0.11.1\"},{\"features\":[\"full\",\"extra-traits\"],\"name\":\"syn\",\"req\":\"^2.0.15\"}],\"features\":{\"diagnostics\":[],\"suggestions\":[\"strsim\"]}}", + "darling_macro_0.23.0": "{\"dependencies\":[{\"name\":\"darling_core\",\"req\":\"=0.23.0\"},{\"name\":\"quote\",\"req\":\"^1.0.18\"},{\"name\":\"syn\",\"req\":\"^2.0.15\"}],\"features\":{}}", "data-encoding_2.11.0": "{\"dependencies\":[],\"features\":{\"alloc\":[],\"default\":[\"std\"],\"std\":[\"alloc\"]}}", "der_0.7.10": "{\"dependencies\":[{\"features\":[\"derive\"],\"name\":\"arbitrary\",\"optional\":true,\"req\":\"^1.3\"},{\"default_features\":false,\"name\":\"bytes\",\"optional\":true,\"req\":\"^1\"},{\"name\":\"const-oid\",\"optional\":true,\"req\":\"^0.9.2\"},{\"name\":\"der_derive\",\"optional\":true,\"req\":\"^0.7.2\"},{\"name\":\"flagset\",\"optional\":true,\"req\":\"^0.4.3\"},{\"kind\":\"dev\",\"name\":\"hex-literal\",\"req\":\"^0.4.1\"},{\"features\":[\"alloc\"],\"name\":\"pem-rfc7468\",\"optional\":true,\"req\":\"^0.7\"},{\"kind\":\"dev\",\"name\":\"proptest\",\"req\":\"^1\"},{\"default_features\":false,\"name\":\"time\",\"optional\":true,\"req\":\"^0.3.4\"},{\"default_features\":false,\"name\":\"zeroize\",\"optional\":true,\"req\":\"^1.5\"}],\"features\":{\"alloc\":[\"zeroize?/alloc\"],\"arbitrary\":[\"dep:arbitrary\",\"const-oid?/arbitrary\",\"std\"],\"bytes\":[\"dep:bytes\",\"alloc\"],\"derive\":[\"dep:der_derive\"],\"oid\":[\"dep:const-oid\"],\"pem\":[\"dep:pem-rfc7468\",\"alloc\",\"zeroize\"],\"real\":[],\"std\":[\"alloc\"]}}", "deranged_0.5.8": "{\"dependencies\":[{\"name\":\"deranged-macros\",\"optional\":true,\"req\":\"=0.3.0\"},{\"default_features\":false,\"name\":\"num-traits\",\"optional\":true,\"req\":\"^0.2.15\"},{\"default_features\":false,\"name\":\"powerfmt\",\"optional\":true,\"req\":\"^0.2.0\"},{\"default_features\":false,\"name\":\"quickcheck\",\"optional\":true,\"req\":\"^1.0.3\"},{\"default_features\":false,\"name\":\"rand010\",\"optional\":true,\"package\":\"rand\",\"req\":\"^0.10.0\"},{\"kind\":\"dev\",\"name\":\"rand010\",\"package\":\"rand\",\"req\":\"^0.10.0\"},{\"default_features\":false,\"name\":\"rand08\",\"optional\":true,\"package\":\"rand\",\"req\":\"^0.8.4\"},{\"kind\":\"dev\",\"name\":\"rand08\",\"package\":\"rand\",\"req\":\"^0.8.4\"},{\"default_features\":false,\"name\":\"rand09\",\"optional\":true,\"package\":\"rand\",\"req\":\"^0.9.0\"},{\"kind\":\"dev\",\"name\":\"rand09\",\"package\":\"rand\",\"req\":\"^0.9.0\"},{\"default_features\":false,\"name\":\"serde_core\",\"optional\":true,\"req\":\"^1.0.220\"},{\"kind\":\"dev\",\"name\":\"serde_json\",\"req\":\"^1.0.86\"}],\"features\":{\"alloc\":[],\"default\":[],\"macros\":[\"dep:deranged-macros\"],\"num\":[\"dep:num-traits\"],\"powerfmt\":[\"dep:powerfmt\"],\"quickcheck\":[\"dep:quickcheck\",\"alloc\"],\"rand\":[\"rand08\",\"rand09\",\"rand010\"],\"rand010\":[\"dep:rand010\"],\"rand08\":[\"dep:rand08\"],\"rand09\":[\"dep:rand09\"],\"serde\":[\"dep:serde_core\"]}}", @@ -915,6 +840,7 @@ "digest_0.10.7": "{\"dependencies\":[{\"name\":\"blobby\",\"optional\":true,\"req\":\"^0.3\"},{\"name\":\"block-buffer\",\"optional\":true,\"req\":\"^0.10\"},{\"name\":\"const-oid\",\"optional\":true,\"req\":\"^0.9\"},{\"name\":\"crypto-common\",\"req\":\"^0.1.3\"},{\"default_features\":false,\"name\":\"subtle\",\"optional\":true,\"req\":\"^2.4\"}],\"features\":{\"alloc\":[],\"core-api\":[\"block-buffer\"],\"default\":[\"core-api\"],\"dev\":[\"blobby\"],\"mac\":[\"subtle\"],\"oid\":[\"const-oid\"],\"rand_core\":[\"crypto-common/rand_core\"],\"std\":[\"alloc\",\"crypto-common/std\"]}}", "dirs-sys_0.5.0": "{\"dependencies\":[{\"name\":\"libc\",\"req\":\"^0.2\",\"target\":\"cfg(unix)\"},{\"name\":\"option-ext\",\"req\":\"^0.2.0\"},{\"default_features\":false,\"name\":\"redox_users\",\"req\":\"^0.5\",\"target\":\"cfg(target_os = \\\"redox\\\")\"},{\"features\":[\"Win32_UI_Shell\",\"Win32_Foundation\",\"Win32_Globalization\",\"Win32_System_Com\"],\"name\":\"windows-sys\",\"req\":\">=0.59.0\",\"target\":\"cfg(windows)\"}],\"features\":{}}", "dirs_6.0.0": "{\"dependencies\":[{\"name\":\"dirs-sys\",\"req\":\"^0.5.0\"}],\"features\":{}}", + "displaydoc_0.2.5": "{\"dependencies\":[{\"default_features\":false,\"kind\":\"dev\",\"name\":\"libc\",\"req\":\"^0.2\"},{\"kind\":\"dev\",\"name\":\"pretty_assertions\",\"req\":\"^0.6.1\"},{\"name\":\"proc-macro2\",\"req\":\"^1.0\"},{\"name\":\"quote\",\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"rustversion\",\"req\":\"^1.0.0\"},{\"kind\":\"dev\",\"name\":\"static_assertions\",\"req\":\"^1.1\"},{\"name\":\"syn\",\"req\":\"^2.0\"},{\"kind\":\"dev\",\"name\":\"thiserror\",\"req\":\"^1.0.24\"},{\"kind\":\"dev\",\"name\":\"trybuild\",\"req\":\"^1.0\"}],\"features\":{\"default\":[\"std\"],\"std\":[]}}", "displaydoc_0.2.6": "{\"dependencies\":[{\"default_features\":false,\"kind\":\"dev\",\"name\":\"libc\",\"req\":\"^0.2\"},{\"kind\":\"dev\",\"name\":\"pretty_assertions\",\"req\":\"^1.4.0\"},{\"name\":\"proc-macro2\",\"req\":\"^1.0\"},{\"name\":\"quote\",\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"rustversion\",\"req\":\"^1.0.0\"},{\"kind\":\"dev\",\"name\":\"static_assertions\",\"req\":\"^1.1\"},{\"name\":\"syn\",\"req\":\"^2.0\"},{\"kind\":\"dev\",\"name\":\"thiserror\",\"req\":\"^1.0.24\"},{\"kind\":\"dev\",\"name\":\"trybuild\",\"req\":\"^1.0\"}],\"features\":{\"default\":[\"std\"],\"std\":[]}}", "dunce_1.0.5": "{\"dependencies\":[],\"features\":{}}", "dyn-clone_1.0.19": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"rustversion\",\"req\":\"^1.0\"},{\"features\":[\"diff\"],\"kind\":\"dev\",\"name\":\"trybuild\",\"req\":\"^1.0.66\"}],\"features\":{}}", @@ -931,12 +857,14 @@ "errno_0.3.14": "{\"dependencies\":[{\"default_features\":false,\"name\":\"libc\",\"req\":\"^0.2\",\"target\":\"cfg(target_os=\\\"hermit\\\")\"},{\"default_features\":false,\"name\":\"libc\",\"req\":\"^0.2\",\"target\":\"cfg(target_os=\\\"wasi\\\")\"},{\"default_features\":false,\"name\":\"libc\",\"req\":\"^0.2\",\"target\":\"cfg(unix)\"},{\"features\":[\"Win32_Foundation\",\"Win32_System_Diagnostics_Debug\"],\"name\":\"windows-sys\",\"req\":\">=0.52, <0.62\",\"target\":\"cfg(windows)\"}],\"features\":{\"default\":[\"std\"],\"std\":[\"libc/std\"]}}", "event-listener-strategy_0.5.4": "{\"dependencies\":[{\"default_features\":false,\"name\":\"event-listener\",\"req\":\"^5.0.0\"},{\"kind\":\"dev\",\"name\":\"futures-lite\",\"req\":\"^2.0.0\"},{\"name\":\"pin-project-lite\",\"req\":\"^0.2.12\"},{\"kind\":\"dev\",\"name\":\"wasm-bindgen-test\",\"req\":\"^0.3.37\",\"target\":\"cfg(target_family = \\\"wasm\\\")\"}],\"features\":{\"default\":[\"std\"],\"loom\":[\"event-listener/loom\"],\"portable-atomic\":[\"event-listener/portable-atomic\"],\"std\":[\"event-listener/std\"]}}", "event-listener_5.4.1": "{\"dependencies\":[{\"default_features\":false,\"name\":\"concurrent-queue\",\"req\":\"^2.4.0\"},{\"default_features\":false,\"features\":[\"cargo_bench_support\"],\"kind\":\"dev\",\"name\":\"criterion\",\"req\":\"^0.7\"},{\"default_features\":false,\"name\":\"critical-section\",\"optional\":true,\"req\":\"^1.2.0\"},{\"features\":[\"std\"],\"kind\":\"dev\",\"name\":\"critical-section\",\"req\":\"^1.2.0\"},{\"kind\":\"dev\",\"name\":\"futures-lite\",\"req\":\"^2.0.0\"},{\"name\":\"loom\",\"optional\":true,\"req\":\"^0.7\",\"target\":\"cfg(loom)\"},{\"name\":\"parking\",\"optional\":true,\"req\":\"^2.0.0\",\"target\":\"cfg(not(target_family = \\\"wasm\\\"))\"},{\"name\":\"pin-project-lite\",\"req\":\"^0.2.12\"},{\"default_features\":false,\"features\":[\"alloc\"],\"name\":\"portable-atomic-util\",\"optional\":true,\"req\":\"^0.2.0\"},{\"default_features\":false,\"name\":\"portable_atomic_crate\",\"optional\":true,\"package\":\"portable-atomic\",\"req\":\"^1.2.0\"},{\"kind\":\"dev\",\"name\":\"try-lock\",\"req\":\"^0.2.5\"},{\"kind\":\"dev\",\"name\":\"waker-fn\",\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"wasm-bindgen-test\",\"req\":\"^0.3\",\"target\":\"cfg(target_family = \\\"wasm\\\")\"}],\"features\":{\"default\":[\"std\"],\"loom\":[\"concurrent-queue/loom\",\"parking?/loom\",\"dep:loom\"],\"portable-atomic\":[\"portable-atomic-util\",\"portable_atomic_crate\",\"concurrent-queue/portable-atomic\"],\"std\":[\"concurrent-queue/std\",\"parking\"]}}", + "fallible-iterator_0.2.0": "{\"dependencies\":[],\"features\":{\"alloc\":[],\"default\":[\"std\"],\"std\":[]}}", "fastcdc_3.2.1": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"aes\",\"req\":\"^0.8.2\"},{\"name\":\"async-stream\",\"optional\":true,\"req\":\"^0.3\"},{\"kind\":\"dev\",\"name\":\"byteorder\",\"req\":\"^1.4.3\"},{\"features\":[\"cargo\"],\"kind\":\"dev\",\"name\":\"clap\",\"req\":\"^4.2.1\"},{\"kind\":\"dev\",\"name\":\"ctr\",\"req\":\"^0.9.2\"},{\"name\":\"futures\",\"optional\":true,\"req\":\"^0.3\"},{\"kind\":\"dev\",\"name\":\"futures-test\",\"req\":\"^0.3\"},{\"kind\":\"dev\",\"name\":\"md-5\",\"req\":\"^0.10.5\"},{\"kind\":\"dev\",\"name\":\"memmap2\",\"req\":\"^0.9.5\"},{\"features\":[\"io-util\"],\"name\":\"tokio\",\"optional\":true,\"req\":\"^1\"},{\"features\":[\"fs\",\"io-util\",\"rt\",\"rt-multi-thread\",\"macros\"],\"kind\":\"dev\",\"name\":\"tokio\",\"req\":\"^1\"},{\"name\":\"tokio-stream\",\"optional\":true,\"req\":\"^0.1\"}],\"features\":{\"default\":[],\"futures\":[\"dep:futures\"],\"tokio\":[\"dep:tokio\",\"tokio-stream\",\"async-stream\"]}}", "fastrand_2.3.0": "{\"dependencies\":[{\"features\":[\"js\"],\"name\":\"getrandom\",\"optional\":true,\"req\":\"^0.2\",\"target\":\"cfg(all(any(target_arch = \\\"wasm32\\\", target_arch = \\\"wasm64\\\"), target_os = \\\"unknown\\\"))\"},{\"kind\":\"dev\",\"name\":\"getrandom\",\"req\":\"^0.2\"},{\"features\":[\"js\"],\"kind\":\"dev\",\"name\":\"getrandom\",\"req\":\"^0.2\",\"target\":\"cfg(all(any(target_arch = \\\"wasm32\\\", target_arch = \\\"wasm64\\\"), target_os = \\\"unknown\\\"))\"},{\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.8\"},{\"kind\":\"dev\",\"name\":\"wasm-bindgen-test\",\"req\":\"^0.3\",\"target\":\"cfg(all(any(target_arch = \\\"wasm32\\\", target_arch = \\\"wasm64\\\"), target_os = \\\"unknown\\\"))\"},{\"kind\":\"dev\",\"name\":\"wyhash\",\"req\":\"^0.5\"}],\"features\":{\"alloc\":[],\"default\":[\"std\"],\"js\":[\"std\",\"getrandom\"],\"std\":[\"alloc\"]}}", "fastrand_2.4.1": "{\"dependencies\":[{\"features\":[\"wasm_js\"],\"name\":\"getrandom\",\"optional\":true,\"req\":\"^0.3.4\",\"target\":\"cfg(all(any(target_arch = \\\"wasm32\\\", target_arch = \\\"wasm64\\\"), target_os = \\\"unknown\\\"))\"},{\"kind\":\"dev\",\"name\":\"getrandom\",\"req\":\"^0.3.4\"},{\"features\":[\"wasm_js\"],\"kind\":\"dev\",\"name\":\"getrandom\",\"req\":\"^0.3.4\",\"target\":\"cfg(all(any(target_arch = \\\"wasm32\\\", target_arch = \\\"wasm64\\\"), target_os = \\\"unknown\\\"))\"},{\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.9\"},{\"kind\":\"dev\",\"name\":\"wasm-bindgen-test\",\"req\":\"^0.3\",\"target\":\"cfg(all(any(target_arch = \\\"wasm32\\\", target_arch = \\\"wasm64\\\"), target_os = \\\"unknown\\\"))\"},{\"kind\":\"dev\",\"name\":\"wyhash\",\"req\":\"^0.6\"}],\"features\":{\"alloc\":[],\"default\":[\"std\"],\"js\":[\"std\",\"getrandom\"],\"std\":[\"alloc\"]}}", "ff_0.13.1": "{\"dependencies\":[{\"default_features\":false,\"name\":\"bitvec\",\"optional\":true,\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"blake2b_simd\",\"req\":\"^1\"},{\"default_features\":false,\"name\":\"byteorder\",\"optional\":true,\"req\":\"^1\"},{\"name\":\"ff_derive\",\"optional\":true,\"req\":\"^0.13.1\"},{\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.8\"},{\"default_features\":false,\"name\":\"rand_core\",\"req\":\"^0.6\"},{\"default_features\":false,\"features\":[\"i128\"],\"name\":\"subtle\",\"req\":\"^2.2.1\"}],\"features\":{\"alloc\":[],\"bits\":[\"bitvec\"],\"default\":[\"bits\",\"std\"],\"derive\":[\"byteorder\",\"ff_derive\"],\"derive_bits\":[\"bits\",\"ff_derive/bits\"],\"std\":[\"alloc\"]}}", "fiat-crypto_0.2.9": "{\"dependencies\":[],\"features\":{\"default\":[\"std\"],\"std\":[]}}", "filetime_0.2.26": "{\"dependencies\":[{\"name\":\"cfg-if\",\"req\":\"^1.0.0\"},{\"name\":\"libc\",\"req\":\"^0.2.27\",\"target\":\"cfg(unix)\"},{\"name\":\"libredox\",\"req\":\"^0.1.0\",\"target\":\"cfg(target_os = \\\"redox\\\")\"},{\"kind\":\"dev\",\"name\":\"tempfile\",\"req\":\"^3\"},{\"features\":[\"Win32_Foundation\",\"Win32_Storage_FileSystem\"],\"name\":\"windows-sys\",\"req\":\"^0.60.0\",\"target\":\"cfg(windows)\"}],\"features\":{}}", + "filetime_0.2.29": "{\"dependencies\":[{\"name\":\"cfg-if\",\"req\":\"^1.0.0\"},{\"name\":\"libc\",\"req\":\"^0.2.27\",\"target\":\"cfg(unix)\"},{\"kind\":\"dev\",\"name\":\"tempfile\",\"req\":\"^3\"}],\"features\":{}}", "find-msvc-tools_0.1.9": "{\"dependencies\":[],\"features\":{}}", "fixedbitset_0.5.7": "{\"dependencies\":[{\"name\":\"serde\",\"optional\":true,\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"serde_json\",\"req\":\"^1.0\"}],\"features\":{\"default\":[\"std\"],\"std\":[]}}", "flate2_1.1.9": "{\"dependencies\":[{\"name\":\"cloudflare-zlib-sys\",\"optional\":true,\"req\":\"^0.3.6\"},{\"name\":\"crc32fast\",\"optional\":true,\"req\":\"^1.2.0\"},{\"name\":\"document-features\",\"optional\":true,\"req\":\"^0.2\"},{\"default_features\":false,\"kind\":\"dev\",\"name\":\"futures\",\"req\":\"^0.3\"},{\"name\":\"libz-ng-sys\",\"optional\":true,\"req\":\"^1.1.16\"},{\"default_features\":false,\"name\":\"libz-sys\",\"optional\":true,\"req\":\"^1.1.20\"},{\"default_features\":false,\"features\":[\"with-alloc\",\"simd\"],\"name\":\"miniz_oxide\",\"req\":\"^0.8.5\",\"target\":\"cfg(all(target_arch = \\\"wasm32\\\", not(target_os = \\\"emscripten\\\")))\"},{\"default_features\":false,\"features\":[\"with-alloc\",\"simd\"],\"name\":\"miniz_oxide\",\"optional\":true,\"req\":\"^0.8.5\"},{\"default_features\":false,\"kind\":\"dev\",\"name\":\"quickcheck\",\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.9\"},{\"default_features\":false,\"features\":[\"std\",\"rust-allocator\"],\"name\":\"zlib-rs\",\"optional\":true,\"req\":\"^0.6.0\"}],\"features\":{\"any_c_zlib\":[\"any_zlib\"],\"any_impl\":[],\"any_zlib\":[\"any_impl\"],\"cloudflare_zlib\":[\"any_c_zlib\",\"cloudflare-zlib-sys\",\"dep:crc32fast\"],\"default\":[\"rust_backend\"],\"miniz-sys\":[\"rust_backend\"],\"miniz_oxide\":[\"any_impl\",\"dep:miniz_oxide\",\"dep:crc32fast\"],\"rust_backend\":[\"miniz_oxide\",\"any_impl\"],\"zlib\":[\"any_c_zlib\",\"libz-sys\",\"dep:crc32fast\"],\"zlib-default\":[\"any_c_zlib\",\"libz-sys/default\",\"dep:crc32fast\"],\"zlib-ng\":[\"any_c_zlib\",\"libz-ng-sys\",\"dep:crc32fast\"],\"zlib-ng-compat\":[\"zlib\",\"libz-sys/zlib-ng\",\"dep:crc32fast\"],\"zlib-rs\":[\"any_zlib\",\"dep:zlib-rs\"]}}", @@ -961,8 +889,10 @@ "gcloud-storage_1.3.0": "{\"dependencies\":[{\"name\":\"anyhow\",\"req\":\"^1.0\"},{\"name\":\"base64\",\"req\":\"^0.22\"},{\"name\":\"bytes\",\"req\":\"^1.5\"},{\"kind\":\"dev\",\"name\":\"ctor\",\"req\":\"^0.5\"},{\"name\":\"futures-util\",\"req\":\"^0.3\"},{\"default_features\":false,\"name\":\"google-cloud-auth\",\"optional\":true,\"package\":\"gcloud-auth\",\"req\":\"^1.3.0\"},{\"name\":\"google-cloud-metadata\",\"optional\":true,\"package\":\"gcloud-metadata\",\"req\":\"^1.0.2\"},{\"name\":\"hex\",\"req\":\"^0.4\"},{\"name\":\"once_cell\",\"req\":\"^1.18\"},{\"name\":\"percent-encoding\",\"req\":\"^2.3\"},{\"features\":[\"pem\"],\"name\":\"pkcs8\",\"req\":\"^0.10\"},{\"name\":\"regex\",\"req\":\"^1.9\"},{\"default_features\":false,\"features\":[\"json\",\"stream\",\"multipart\"],\"name\":\"reqwest\",\"req\":\"^0.13\"},{\"features\":[\"json\",\"multipart\",\"query\"],\"name\":\"reqwest-middleware\",\"req\":\"^0.5\"},{\"name\":\"ring\",\"req\":\"^0.17\"},{\"features\":[\"derive\"],\"name\":\"serde\",\"req\":\"^1.0\"},{\"name\":\"serde_json\",\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"serial_test\",\"req\":\"^3.1\"},{\"name\":\"sha2\",\"req\":\"^0.10\"},{\"name\":\"thiserror\",\"req\":\"^2.0\"},{\"features\":[\"std\",\"macros\",\"formatting\",\"parsing\",\"serde\"],\"name\":\"time\",\"req\":\"^0.3\"},{\"name\":\"token-source\",\"req\":\"^1.0\"},{\"features\":[\"macros\"],\"name\":\"tokio\",\"req\":\"^1.32\"},{\"features\":[\"rt-multi-thread\"],\"kind\":\"dev\",\"name\":\"tokio\",\"req\":\"^1.32\"},{\"name\":\"tracing\",\"req\":\"^0.1\"},{\"features\":[\"env-filter\"],\"kind\":\"dev\",\"name\":\"tracing-subscriber\",\"req\":\"^0.3.17\"},{\"name\":\"url\",\"req\":\"^2.4\"}],\"features\":{\"auth\":[\"google-cloud-auth\",\"google-cloud-metadata\"],\"default\":[\"rustls-tls\",\"auth\",\"jwt-aws-lc-rs\"],\"external-account\":[\"google-cloud-auth?/external-account\"],\"hickory-dns\":[\"reqwest/hickory-dns\",\"google-cloud-auth?/hickory-dns\"],\"jwt-aws-lc-rs\":[\"google-cloud-auth?/jwt-aws-lc-rs\"],\"jwt-rust-crypto\":[\"google-cloud-auth?/jwt-rust-crypto\"],\"native-tls\":[\"google-cloud-auth?/native-tls\"],\"rustls-no-provider\":[\"google-cloud-auth?/rustls-no-provider\"],\"rustls-tls\":[\"google-cloud-auth?/rustls-tls\"],\"trace\":[]}}", "generic-array_0.14.9": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"bincode\",\"req\":\"^1.0\"},{\"default_features\":false,\"name\":\"serde\",\"optional\":true,\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"serde_json\",\"req\":\"^1.0\"},{\"name\":\"typenum\",\"req\":\"^1.12\"},{\"kind\":\"build\",\"name\":\"version_check\",\"req\":\"^0.9\"},{\"default_features\":false,\"name\":\"zeroize\",\"optional\":true,\"req\":\"^1\"}],\"features\":{\"more_lengths\":[]}}", "getrandom_0.2.16": "{\"dependencies\":[{\"name\":\"cfg-if\",\"req\":\"^1\"},{\"name\":\"compiler_builtins\",\"optional\":true,\"req\":\"^0.1\"},{\"name\":\"core\",\"optional\":true,\"package\":\"rustc-std-workspace-core\",\"req\":\"^1.0\"},{\"name\":\"js-sys\",\"optional\":true,\"req\":\"^0.3\",\"target\":\"cfg(all(any(target_arch = \\\"wasm32\\\", target_arch = \\\"wasm64\\\"), target_os = \\\"unknown\\\"))\"},{\"default_features\":false,\"name\":\"libc\",\"req\":\"^0.2.154\",\"target\":\"cfg(unix)\"},{\"default_features\":false,\"name\":\"wasi\",\"req\":\"^0.11\",\"target\":\"cfg(target_os = \\\"wasi\\\")\"},{\"default_features\":false,\"name\":\"wasm-bindgen\",\"optional\":true,\"req\":\"^0.2.62\",\"target\":\"cfg(all(any(target_arch = \\\"wasm32\\\", target_arch = \\\"wasm64\\\"), target_os = \\\"unknown\\\"))\"},{\"kind\":\"dev\",\"name\":\"wasm-bindgen-test\",\"req\":\"^0.3.18\",\"target\":\"cfg(all(any(target_arch = \\\"wasm32\\\", target_arch = \\\"wasm64\\\"), target_os = \\\"unknown\\\"))\"}],\"features\":{\"custom\":[],\"js\":[\"wasm-bindgen\",\"js-sys\"],\"linux_disable_fallback\":[],\"rdrand\":[],\"rustc-dep-of-std\":[\"compiler_builtins\",\"core\",\"libc/rustc-dep-of-std\",\"wasi/rustc-dep-of-std\"],\"std\":[],\"test-in-browser\":[]}}", + "getrandom_0.2.17": "{\"dependencies\":[{\"name\":\"cfg-if\",\"req\":\"^1\"},{\"name\":\"compiler_builtins\",\"optional\":true,\"req\":\"^0.1\"},{\"name\":\"core\",\"optional\":true,\"package\":\"rustc-std-workspace-core\",\"req\":\"^1.0\"},{\"name\":\"js-sys\",\"optional\":true,\"req\":\"^0.3\",\"target\":\"cfg(all(any(target_arch = \\\"wasm32\\\", target_arch = \\\"wasm64\\\"), target_os = \\\"unknown\\\"))\"},{\"default_features\":false,\"name\":\"libc\",\"req\":\"^0.2.154\",\"target\":\"cfg(unix)\"},{\"default_features\":false,\"name\":\"wasi\",\"req\":\"^0.11\",\"target\":\"cfg(target_os = \\\"wasi\\\")\"},{\"default_features\":false,\"name\":\"wasm-bindgen\",\"optional\":true,\"req\":\"^0.2.62\",\"target\":\"cfg(all(any(target_arch = \\\"wasm32\\\", target_arch = \\\"wasm64\\\"), target_os = \\\"unknown\\\"))\"},{\"kind\":\"dev\",\"name\":\"wasm-bindgen-test\",\"req\":\"^0.3.18\",\"target\":\"cfg(all(any(target_arch = \\\"wasm32\\\", target_arch = \\\"wasm64\\\"), target_os = \\\"unknown\\\"))\"}],\"features\":{\"custom\":[],\"js\":[\"wasm-bindgen\",\"js-sys\"],\"linux_disable_fallback\":[],\"rdrand\":[],\"rustc-dep-of-std\":[\"compiler_builtins\",\"core\",\"libc/rustc-dep-of-std\",\"wasi/rustc-dep-of-std\"],\"std\":[],\"test-in-browser\":[]}}", "getrandom_0.3.4": "{\"dependencies\":[{\"name\":\"cfg-if\",\"req\":\"^1\"},{\"default_features\":false,\"name\":\"js-sys\",\"optional\":true,\"req\":\"^0.3.77\",\"target\":\"cfg(all(target_arch = \\\"wasm32\\\", any(target_os = \\\"unknown\\\", target_os = \\\"none\\\"), target_feature = \\\"atomics\\\"))\"},{\"default_features\":false,\"name\":\"libc\",\"req\":\"^0.2.154\",\"target\":\"cfg(all(any(target_os = \\\"linux\\\", target_os = \\\"android\\\"), not(any(all(target_os = \\\"linux\\\", target_env = \\\"\\\"), getrandom_backend = \\\"custom\\\", getrandom_backend = \\\"linux_raw\\\", getrandom_backend = \\\"rdrand\\\", getrandom_backend = \\\"rndr\\\"))))\"},{\"default_features\":false,\"name\":\"libc\",\"req\":\"^0.2.154\",\"target\":\"cfg(any(target_os = \\\"dragonfly\\\", target_os = \\\"freebsd\\\", target_os = \\\"hurd\\\", target_os = \\\"illumos\\\", target_os = \\\"cygwin\\\", all(target_os = \\\"horizon\\\", target_arch = \\\"arm\\\")))\"},{\"default_features\":false,\"name\":\"libc\",\"req\":\"^0.2.154\",\"target\":\"cfg(any(target_os = \\\"haiku\\\", target_os = \\\"redox\\\", target_os = \\\"nto\\\", target_os = \\\"aix\\\"))\"},{\"default_features\":false,\"name\":\"libc\",\"req\":\"^0.2.154\",\"target\":\"cfg(any(target_os = \\\"ios\\\", target_os = \\\"visionos\\\", target_os = \\\"watchos\\\", target_os = \\\"tvos\\\"))\"},{\"default_features\":false,\"name\":\"libc\",\"req\":\"^0.2.154\",\"target\":\"cfg(any(target_os = \\\"macos\\\", target_os = \\\"openbsd\\\", target_os = \\\"vita\\\", target_os = \\\"emscripten\\\"))\"},{\"default_features\":false,\"name\":\"libc\",\"req\":\"^0.2.154\",\"target\":\"cfg(target_os = \\\"netbsd\\\")\"},{\"default_features\":false,\"name\":\"libc\",\"req\":\"^0.2.154\",\"target\":\"cfg(target_os = \\\"solaris\\\")\"},{\"default_features\":false,\"name\":\"libc\",\"req\":\"^0.2.154\",\"target\":\"cfg(target_os = \\\"vxworks\\\")\"},{\"default_features\":false,\"name\":\"r-efi\",\"req\":\"^5.1\",\"target\":\"cfg(all(target_os = \\\"uefi\\\", getrandom_backend = \\\"efi_rng\\\"))\"},{\"default_features\":false,\"name\":\"wasip2\",\"req\":\"^1\",\"target\":\"cfg(all(target_arch = \\\"wasm32\\\", target_os = \\\"wasi\\\", target_env = \\\"p2\\\"))\"},{\"default_features\":false,\"name\":\"wasm-bindgen\",\"optional\":true,\"req\":\"^0.2.98\",\"target\":\"cfg(all(target_arch = \\\"wasm32\\\", any(target_os = \\\"unknown\\\", target_os = \\\"none\\\")))\"},{\"kind\":\"dev\",\"name\":\"wasm-bindgen-test\",\"req\":\"^0.3\",\"target\":\"cfg(all(target_arch = \\\"wasm32\\\", any(target_os = \\\"unknown\\\", target_os = \\\"none\\\")))\"}],\"features\":{\"std\":[],\"wasm_js\":[\"dep:wasm-bindgen\",\"dep:js-sys\"]}}", "getrandom_0.4.2": "{\"dependencies\":[{\"name\":\"cfg-if\",\"req\":\"^1\"},{\"default_features\":false,\"name\":\"js-sys\",\"optional\":true,\"req\":\"^0.3.77\",\"target\":\"cfg(all(target_arch = \\\"wasm32\\\", any(target_os = \\\"unknown\\\", target_os = \\\"none\\\"), target_feature = \\\"atomics\\\"))\"},{\"default_features\":false,\"name\":\"libc\",\"req\":\"^0.2.154\",\"target\":\"cfg(all(any(target_os = \\\"linux\\\", target_os = \\\"android\\\"), not(any(all(target_os = \\\"linux\\\", target_env = \\\"\\\"), getrandom_backend = \\\"custom\\\", getrandom_backend = \\\"linux_raw\\\", getrandom_backend = \\\"rdrand\\\", getrandom_backend = \\\"rndr\\\"))))\"},{\"default_features\":false,\"name\":\"libc\",\"req\":\"^0.2.154\",\"target\":\"cfg(any(target_os = \\\"dragonfly\\\", target_os = \\\"freebsd\\\", target_os = \\\"hurd\\\", target_os = \\\"illumos\\\", target_os = \\\"cygwin\\\", all(target_os = \\\"horizon\\\", target_arch = \\\"arm\\\")))\"},{\"default_features\":false,\"name\":\"libc\",\"req\":\"^0.2.154\",\"target\":\"cfg(any(target_os = \\\"haiku\\\", target_os = \\\"redox\\\", target_os = \\\"nto\\\", target_os = \\\"aix\\\"))\"},{\"default_features\":false,\"name\":\"libc\",\"req\":\"^0.2.154\",\"target\":\"cfg(any(target_os = \\\"ios\\\", target_os = \\\"visionos\\\", target_os = \\\"watchos\\\", target_os = \\\"tvos\\\"))\"},{\"default_features\":false,\"name\":\"libc\",\"req\":\"^0.2.154\",\"target\":\"cfg(any(target_os = \\\"macos\\\", target_os = \\\"openbsd\\\", target_os = \\\"vita\\\", target_os = \\\"emscripten\\\"))\"},{\"default_features\":false,\"name\":\"libc\",\"req\":\"^0.2.154\",\"target\":\"cfg(target_os = \\\"netbsd\\\")\"},{\"default_features\":false,\"name\":\"libc\",\"req\":\"^0.2.154\",\"target\":\"cfg(target_os = \\\"solaris\\\")\"},{\"default_features\":false,\"name\":\"libc\",\"req\":\"^0.2.154\",\"target\":\"cfg(target_os = \\\"vxworks\\\")\"},{\"default_features\":false,\"name\":\"r-efi\",\"req\":\"^6\",\"target\":\"cfg(all(target_os = \\\"uefi\\\", getrandom_backend = \\\"efi_rng\\\"))\"},{\"name\":\"rand_core\",\"optional\":true,\"req\":\"^0.10.0\"},{\"default_features\":false,\"name\":\"wasip2\",\"req\":\"^1\",\"target\":\"cfg(all(target_arch = \\\"wasm32\\\", target_os = \\\"wasi\\\", target_env = \\\"p2\\\"))\"},{\"name\":\"wasip3\",\"req\":\"^0.4\",\"target\":\"cfg(all(target_arch = \\\"wasm32\\\", target_os = \\\"wasi\\\", target_env = \\\"p3\\\"))\"},{\"default_features\":false,\"name\":\"wasm-bindgen\",\"optional\":true,\"req\":\"^0.2.98\",\"target\":\"cfg(all(target_arch = \\\"wasm32\\\", any(target_os = \\\"unknown\\\", target_os = \\\"none\\\")))\"},{\"kind\":\"dev\",\"name\":\"wasm-bindgen-test\",\"req\":\"^0.3\",\"target\":\"cfg(all(target_arch = \\\"wasm32\\\", any(target_os = \\\"unknown\\\", target_os = \\\"none\\\")))\"}],\"features\":{\"std\":[],\"sys_rng\":[\"dep:rand_core\"],\"wasm_js\":[\"dep:wasm-bindgen\",\"dep:js-sys\"]}}", + "gimli_0.26.2": "{\"dependencies\":[{\"name\":\"alloc\",\"optional\":true,\"package\":\"rustc-std-workspace-alloc\",\"req\":\"^1.0.0\"},{\"name\":\"compiler_builtins\",\"optional\":true,\"req\":\"^0.1.2\"},{\"name\":\"core\",\"optional\":true,\"package\":\"rustc-std-workspace-core\",\"req\":\"^1.0.0\"},{\"kind\":\"dev\",\"name\":\"crossbeam\",\"req\":\"^0.8\"},{\"default_features\":false,\"name\":\"fallible-iterator\",\"optional\":true,\"req\":\"^0.2.0\"},{\"kind\":\"dev\",\"name\":\"getopts\",\"req\":\"^0.2\"},{\"name\":\"indexmap\",\"optional\":true,\"req\":\"^1.0.2\"},{\"kind\":\"dev\",\"name\":\"memmap2\",\"req\":\"^0.5.5\"},{\"kind\":\"dev\",\"name\":\"num_cpus\",\"req\":\"^1\"},{\"features\":[\"wasm\"],\"kind\":\"dev\",\"name\":\"object\",\"req\":\"^0.29.0\"},{\"kind\":\"dev\",\"name\":\"rayon\",\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"regex\",\"req\":\"^1\"},{\"default_features\":false,\"name\":\"stable_deref_trait\",\"optional\":true,\"req\":\"^1.1.0\"},{\"kind\":\"dev\",\"name\":\"test-assembler\",\"req\":\"^0.1.3\"},{\"kind\":\"dev\",\"name\":\"typed-arena\",\"req\":\"^2\"}],\"features\":{\"default\":[\"read\",\"write\",\"std\",\"fallible-iterator\",\"endian-reader\"],\"endian-reader\":[\"read\",\"stable_deref_trait\"],\"read\":[\"read-core\"],\"read-core\":[],\"rustc-dep-of-std\":[\"core\",\"alloc\",\"compiler_builtins\"],\"std\":[\"fallible-iterator/std\",\"stable_deref_trait/std\"],\"write\":[\"indexmap\"]}}", "git+https://github.com/mstyura/ginepro?rev=d08cdeff6300edfb46204b3b9fbde3f3355db35f#d08cdeff6300edfb46204b3b9fbde3f3355db35f_ginepro": "{\"dependencies\":[{\"name\":\"anyhow\"},{\"name\":\"async-trait\"},{\"default_features\":true,\"features\":[\"tokio\"],\"name\":\"hickory-resolver\",\"optional\":false},{\"name\":\"http\"},{\"name\":\"thiserror\"},{\"default_features\":true,\"features\":[\"full\"],\"name\":\"tokio\",\"optional\":false},{\"default_features\":false,\"features\":[\"router\",\"transport\",\"codegen\"],\"name\":\"tonic\",\"optional\":false},{\"default_features\":false,\"features\":[\"discover\"],\"name\":\"tower\",\"optional\":false},{\"name\":\"tracing\"}],\"features\":{\"default\":[\"tls-ring\"],\"tls-aws-lc\":[\"tonic/tls-aws-lc\"],\"tls-ring\":[\"tonic/tls-ring\"]},\"strip_prefix\":\"ginepro\"}", "goblin_0.10.5": "{\"dependencies\":[{\"default_features\":false,\"name\":\"log\",\"optional\":true,\"req\":\"^0.4\"},{\"name\":\"plain\",\"req\":\"^0.2.3\"},{\"default_features\":false,\"name\":\"scroll\",\"req\":\"^0.13\"},{\"kind\":\"dev\",\"name\":\"stderrlog\",\"req\":\"^0.6.0\"}],\"features\":{\"alloc\":[\"scroll/derive\",\"log\"],\"archive\":[\"alloc\"],\"default\":[\"std\",\"elf32\",\"elf64\",\"mach32\",\"mach64\",\"pe32\",\"pe64\",\"te\",\"archive\",\"endian_fd\"],\"elf32\":[],\"elf64\":[],\"endian_fd\":[\"alloc\"],\"mach32\":[\"alloc\",\"endian_fd\",\"archive\"],\"mach64\":[\"alloc\",\"endian_fd\",\"archive\"],\"pe32\":[\"alloc\",\"endian_fd\"],\"pe64\":[\"alloc\",\"endian_fd\"],\"std\":[\"alloc\",\"scroll/std\"],\"te\":[\"alloc\",\"endian_fd\"]}}", "group_0.13.0": "{\"dependencies\":[{\"default_features\":false,\"name\":\"ff\",\"req\":\"^0.13\"},{\"name\":\"memuse\",\"optional\":true,\"req\":\"^0.2\"},{\"default_features\":false,\"name\":\"rand\",\"optional\":true,\"req\":\"^0.8\"},{\"default_features\":false,\"name\":\"rand_core\",\"req\":\"^0.6\"},{\"name\":\"rand_xorshift\",\"optional\":true,\"req\":\"^0.3\"},{\"default_features\":false,\"name\":\"subtle\",\"req\":\"^2.2.1\"}],\"features\":{\"alloc\":[],\"default\":[\"alloc\"],\"tests\":[\"alloc\",\"rand\",\"rand_xorshift\"],\"wnaf-memuse\":[\"alloc\",\"memuse\"]}}", @@ -970,10 +900,12 @@ "h2_0.4.14": "{\"dependencies\":[{\"name\":\"atomic-waker\",\"req\":\"^1.0.0\"},{\"name\":\"bytes\",\"req\":\"^1\"},{\"default_features\":false,\"kind\":\"dev\",\"name\":\"env_logger\",\"req\":\"^0.10\"},{\"name\":\"fnv\",\"req\":\"^1.0.5\"},{\"default_features\":false,\"name\":\"futures-core\",\"req\":\"^0.3\"},{\"default_features\":false,\"name\":\"futures-sink\",\"req\":\"^0.3\"},{\"kind\":\"dev\",\"name\":\"hex\",\"req\":\"^0.4.3\"},{\"name\":\"http\",\"req\":\"^1\"},{\"features\":[\"std\"],\"name\":\"indexmap\",\"req\":\"^2\"},{\"default_features\":false,\"kind\":\"dev\",\"name\":\"quickcheck\",\"req\":\"^1.0.3\"},{\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.8.4\"},{\"kind\":\"dev\",\"name\":\"serde\",\"req\":\"^1.0.0\"},{\"kind\":\"dev\",\"name\":\"serde_json\",\"req\":\"^1.0.0\"},{\"name\":\"slab\",\"req\":\"^0.4.2\"},{\"features\":[\"io-util\"],\"name\":\"tokio\",\"req\":\"^1\"},{\"features\":[\"rt-multi-thread\",\"macros\",\"sync\",\"net\"],\"kind\":\"dev\",\"name\":\"tokio\",\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"tokio-rustls\",\"req\":\"^0.26\"},{\"features\":[\"codec\",\"io\"],\"name\":\"tokio-util\",\"req\":\"^0.7.1\"},{\"default_features\":false,\"features\":[\"std\"],\"name\":\"tracing\",\"req\":\"^0.1.35\"},{\"kind\":\"dev\",\"name\":\"walkdir\",\"req\":\"^2.3.2\"},{\"kind\":\"dev\",\"name\":\"webpki-roots\",\"req\":\"^1\"}],\"features\":{\"stream\":[],\"unstable\":[]}}", "half_2.7.1": "{\"dependencies\":[{\"features\":[\"derive\"],\"name\":\"arbitrary\",\"optional\":true,\"req\":\"^1.4.1\"},{\"default_features\":false,\"features\":[\"derive\"],\"name\":\"bytemuck\",\"optional\":true,\"req\":\"^1.4.1\"},{\"name\":\"cfg-if\",\"req\":\"^1.0.0\"},{\"kind\":\"dev\",\"name\":\"criterion\",\"req\":\"^0.5\"},{\"name\":\"crunchy\",\"req\":\"^0.2.2\",\"target\":\"cfg(target_arch = \\\"spirv\\\")\"},{\"kind\":\"dev\",\"name\":\"crunchy\",\"req\":\"^0.2.2\"},{\"default_features\":false,\"features\":[\"libm\"],\"name\":\"num-traits\",\"optional\":true,\"req\":\"^0.2.16\"},{\"kind\":\"dev\",\"name\":\"quickcheck\",\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"quickcheck_macros\",\"req\":\"^1.0\"},{\"default_features\":false,\"features\":[\"thread_rng\"],\"name\":\"rand\",\"optional\":true,\"req\":\"^0.9.0\"},{\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.9.0\"},{\"default_features\":false,\"name\":\"rand_distr\",\"optional\":true,\"req\":\"^0.5.0\"},{\"name\":\"rkyv\",\"optional\":true,\"req\":\"^0.8.0\"},{\"default_features\":false,\"features\":[\"derive\"],\"name\":\"serde\",\"optional\":true,\"req\":\"^1.0\"},{\"default_features\":false,\"features\":[\"derive\",\"simd\"],\"name\":\"zerocopy\",\"req\":\"^0.8.26\"}],\"features\":{\"alloc\":[],\"default\":[\"std\"],\"nightly\":[],\"rand_distr\":[\"dep:rand\",\"dep:rand_distr\"],\"std\":[\"alloc\"],\"use-intrinsics\":[],\"zerocopy\":[]}}", "hashbrown_0.12.3": "{\"dependencies\":[{\"default_features\":false,\"name\":\"ahash\",\"optional\":true,\"req\":\"^0.7.0\"},{\"name\":\"alloc\",\"optional\":true,\"package\":\"rustc-std-workspace-alloc\",\"req\":\"^1.0.0\"},{\"name\":\"bumpalo\",\"optional\":true,\"req\":\"^3.5.0\"},{\"name\":\"compiler_builtins\",\"optional\":true,\"req\":\"^0.1.2\"},{\"name\":\"core\",\"optional\":true,\"package\":\"rustc-std-workspace-core\",\"req\":\"^1.0.0\"},{\"kind\":\"dev\",\"name\":\"doc-comment\",\"req\":\"^0.3.1\"},{\"kind\":\"dev\",\"name\":\"fnv\",\"req\":\"^1.0.7\"},{\"kind\":\"dev\",\"name\":\"lazy_static\",\"req\":\"^1.4\"},{\"features\":[\"small_rng\"],\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.8.3\"},{\"name\":\"rayon\",\"optional\":true,\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"rayon\",\"req\":\"^1.0\"},{\"default_features\":false,\"name\":\"serde\",\"optional\":true,\"req\":\"^1.0.25\"},{\"kind\":\"dev\",\"name\":\"serde_test\",\"req\":\"^1.0\"}],\"features\":{\"ahash-compile-time-rng\":[\"ahash/compile-time-rng\"],\"default\":[\"ahash\",\"inline-more\"],\"inline-more\":[],\"nightly\":[],\"raw\":[],\"rustc-dep-of-std\":[\"nightly\",\"core\",\"compiler_builtins\",\"alloc\",\"rustc-internal-api\"],\"rustc-internal-api\":[]}}", + "hashbrown_0.14.5": "{\"dependencies\":[{\"default_features\":false,\"name\":\"ahash\",\"optional\":true,\"req\":\"^0.8.7\"},{\"name\":\"alloc\",\"optional\":true,\"package\":\"rustc-std-workspace-alloc\",\"req\":\"^1.0.0\"},{\"default_features\":false,\"features\":[\"alloc\"],\"name\":\"allocator-api2\",\"optional\":true,\"req\":\"^0.2.9\"},{\"features\":[\"allocator-api2\"],\"kind\":\"dev\",\"name\":\"bumpalo\",\"req\":\"^3.13.0\"},{\"name\":\"compiler_builtins\",\"optional\":true,\"req\":\"^0.1.2\"},{\"name\":\"core\",\"optional\":true,\"package\":\"rustc-std-workspace-core\",\"req\":\"^1.0.0\"},{\"kind\":\"dev\",\"name\":\"doc-comment\",\"req\":\"^0.3.1\"},{\"default_features\":false,\"name\":\"equivalent\",\"optional\":true,\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"fnv\",\"req\":\"^1.0.7\"},{\"kind\":\"dev\",\"name\":\"lazy_static\",\"req\":\"^1.4\"},{\"features\":[\"small_rng\"],\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.8.3\"},{\"name\":\"rayon\",\"optional\":true,\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"rayon\",\"req\":\"^1.0\"},{\"default_features\":false,\"features\":[\"alloc\"],\"name\":\"rkyv\",\"optional\":true,\"req\":\"^0.7.42\"},{\"features\":[\"validation\"],\"kind\":\"dev\",\"name\":\"rkyv\",\"req\":\"^0.7.42\"},{\"default_features\":false,\"name\":\"serde\",\"optional\":true,\"req\":\"^1.0.25\"},{\"kind\":\"dev\",\"name\":\"serde_test\",\"req\":\"^1.0\"}],\"features\":{\"default\":[\"ahash\",\"inline-more\",\"allocator-api2\"],\"inline-more\":[],\"nightly\":[\"allocator-api2?/nightly\",\"bumpalo/allocator_api\"],\"raw\":[],\"rustc-dep-of-std\":[\"nightly\",\"core\",\"compiler_builtins\",\"alloc\",\"rustc-internal-api\"],\"rustc-internal-api\":[]}}", "hashbrown_0.15.5": "{\"dependencies\":[{\"name\":\"alloc\",\"optional\":true,\"package\":\"rustc-std-workspace-alloc\",\"req\":\"^1.0.0\"},{\"default_features\":false,\"features\":[\"alloc\"],\"name\":\"allocator-api2\",\"optional\":true,\"req\":\"^0.2.9\"},{\"features\":[\"allocator-api2\"],\"kind\":\"dev\",\"name\":\"bumpalo\",\"req\":\"^3.13.0\"},{\"name\":\"core\",\"optional\":true,\"package\":\"rustc-std-workspace-core\",\"req\":\"^1.0.0\"},{\"kind\":\"dev\",\"name\":\"doc-comment\",\"req\":\"^0.3.1\"},{\"default_features\":false,\"name\":\"equivalent\",\"optional\":true,\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"fnv\",\"req\":\"^1.0.7\"},{\"default_features\":false,\"name\":\"foldhash\",\"optional\":true,\"req\":\"^0.1.2\"},{\"kind\":\"dev\",\"name\":\"lazy_static\",\"req\":\"^1.4\"},{\"features\":[\"small_rng\"],\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.9.0\"},{\"name\":\"rayon\",\"optional\":true,\"req\":\"^1.2\"},{\"kind\":\"dev\",\"name\":\"rayon\",\"req\":\"^1.2\"},{\"default_features\":false,\"name\":\"serde\",\"optional\":true,\"req\":\"^1.0.25\"},{\"kind\":\"dev\",\"name\":\"serde_test\",\"req\":\"^1.0\"}],\"features\":{\"default\":[\"default-hasher\",\"inline-more\",\"allocator-api2\",\"equivalent\",\"raw-entry\"],\"default-hasher\":[\"dep:foldhash\"],\"inline-more\":[],\"nightly\":[\"bumpalo/allocator_api\"],\"raw-entry\":[],\"rustc-dep-of-std\":[\"nightly\",\"core\",\"alloc\",\"rustc-internal-api\"],\"rustc-internal-api\":[]}}", "hashbrown_0.16.1": "{\"dependencies\":[{\"name\":\"alloc\",\"optional\":true,\"package\":\"rustc-std-workspace-alloc\",\"req\":\"^1.0.0\"},{\"default_features\":false,\"features\":[\"alloc\"],\"name\":\"allocator-api2\",\"optional\":true,\"req\":\"^0.2.9\"},{\"features\":[\"allocator-api2\"],\"kind\":\"dev\",\"name\":\"bumpalo\",\"req\":\"^3.13.0\"},{\"name\":\"core\",\"optional\":true,\"package\":\"rustc-std-workspace-core\",\"req\":\"^1.0.0\"},{\"default_features\":false,\"name\":\"equivalent\",\"optional\":true,\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"fnv\",\"req\":\"^1.0.7\"},{\"default_features\":false,\"name\":\"foldhash\",\"optional\":true,\"req\":\"^0.2.0\"},{\"kind\":\"dev\",\"name\":\"lazy_static\",\"req\":\"^1.4\"},{\"kind\":\"dev\",\"name\":\"libc\",\"req\":\"^0.2.155\",\"target\":\"cfg(unix)\"},{\"features\":[\"small_rng\"],\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.9.0\"},{\"name\":\"rayon\",\"optional\":true,\"req\":\"^1.9.0\"},{\"kind\":\"dev\",\"name\":\"rayon\",\"req\":\"^1.2\"},{\"default_features\":false,\"name\":\"serde\",\"optional\":true,\"req\":\"^1.0.220\",\"target\":\"cfg(any())\"},{\"default_features\":false,\"name\":\"serde_core\",\"optional\":true,\"req\":\"^1.0.221\"},{\"kind\":\"dev\",\"name\":\"serde_test\",\"req\":\"^1.0\"}],\"features\":{\"default\":[\"default-hasher\",\"inline-more\",\"allocator-api2\",\"equivalent\",\"raw-entry\"],\"default-hasher\":[\"dep:foldhash\"],\"inline-more\":[],\"nightly\":[\"foldhash?/nightly\",\"bumpalo/allocator_api\"],\"raw-entry\":[],\"rustc-dep-of-std\":[\"nightly\",\"core\",\"alloc\",\"rustc-internal-api\"],\"rustc-internal-api\":[],\"serde\":[\"dep:serde_core\",\"dep:serde\"]}}", "hashbrown_0.17.1": "{\"dependencies\":[{\"name\":\"alloc\",\"optional\":true,\"package\":\"rustc-std-workspace-alloc\",\"req\":\"^1.0.0\"},{\"default_features\":false,\"features\":[\"alloc\"],\"name\":\"allocator-api2\",\"optional\":true,\"req\":\"^0.2.9\"},{\"features\":[\"allocator-api2\"],\"kind\":\"dev\",\"name\":\"bumpalo\",\"req\":\"^3.13.0\"},{\"name\":\"core\",\"optional\":true,\"package\":\"rustc-std-workspace-core\",\"req\":\"^1.0.0\"},{\"features\":[\"html_reports\"],\"kind\":\"dev\",\"name\":\"criterion\",\"req\":\"^0.7\"},{\"default_features\":false,\"name\":\"equivalent\",\"optional\":true,\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"fnv\",\"req\":\"^1.0.7\"},{\"default_features\":false,\"name\":\"foldhash\",\"optional\":true,\"req\":\"^0.2.0\"},{\"kind\":\"dev\",\"name\":\"libc\",\"req\":\"^0.2.155\",\"target\":\"cfg(unix)\"},{\"features\":[\"small_rng\"],\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.9.0\"},{\"name\":\"rayon\",\"optional\":true,\"req\":\"^1.9.0\"},{\"kind\":\"dev\",\"name\":\"rayon\",\"req\":\"^1.2\"},{\"default_features\":false,\"name\":\"serde\",\"optional\":true,\"req\":\"^1.0.220\",\"target\":\"cfg(any())\"},{\"default_features\":false,\"name\":\"serde_core\",\"optional\":true,\"req\":\"^1.0.221\"},{\"kind\":\"dev\",\"name\":\"serde_test\",\"req\":\"^1.0\"}],\"features\":{\"default\":[\"default-hasher\",\"inline-more\",\"allocator-api2\",\"equivalent\",\"raw-entry\"],\"default-hasher\":[\"dep:foldhash\"],\"inline-more\":[],\"nightly\":[\"foldhash?/nightly\",\"bumpalo/allocator_api\"],\"raw-entry\":[],\"rustc-dep-of-std\":[\"nightly\",\"core\",\"alloc\",\"rustc-internal-api\"],\"rustc-internal-api\":[],\"serde\":[\"dep:serde_core\",\"dep:serde\"]}}", "heck_0.5.0": "{\"dependencies\":[],\"features\":{}}", + "hermit-abi_0.5.2": "{\"dependencies\":[{\"name\":\"alloc\",\"optional\":true,\"package\":\"rustc-std-workspace-alloc\",\"req\":\"^1.0.0\"},{\"name\":\"core\",\"optional\":true,\"package\":\"rustc-std-workspace-core\",\"req\":\"^1.0.0\"}],\"features\":{\"default\":[],\"rustc-dep-of-std\":[\"core\",\"alloc\"]}}", "hex_0.4.3": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"criterion\",\"req\":\"^0.3\"},{\"kind\":\"dev\",\"name\":\"faster-hex\",\"req\":\"^0.5\"},{\"kind\":\"dev\",\"name\":\"pretty_assertions\",\"req\":\"^0.6\"},{\"kind\":\"dev\",\"name\":\"rustc-hex\",\"req\":\"^2.1\"},{\"default_features\":false,\"name\":\"serde\",\"optional\":true,\"req\":\"^1.0\"},{\"features\":[\"derive\"],\"kind\":\"dev\",\"name\":\"serde\",\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"serde_json\",\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"version-sync\",\"req\":\"^0.9\"}],\"features\":{\"alloc\":[],\"default\":[\"std\"],\"std\":[\"alloc\"]}}", "hickory-net_0.26.1": "{\"dependencies\":[{\"name\":\"async-trait\",\"req\":\"^0.1.43\"},{\"default_features\":false,\"features\":[\"prebuilt-nasm\"],\"name\":\"aws-lc-rs\",\"optional\":true,\"req\":\"^1.12.3\"},{\"name\":\"bitflags\",\"optional\":true,\"req\":\"^2.4.1\"},{\"name\":\"bytes\",\"optional\":true,\"req\":\"^1\"},{\"name\":\"cfg-if\",\"req\":\"^1\"},{\"default_features\":false,\"features\":[\"alloc\"],\"name\":\"data-encoding\",\"req\":\"^2.2.0\"},{\"default_features\":false,\"features\":[\"alloc\",\"std\"],\"name\":\"futures-channel\",\"req\":\"^0.3.5\"},{\"default_features\":false,\"features\":[\"std\"],\"kind\":\"dev\",\"name\":\"futures-executor\",\"req\":\"^0.3.5\"},{\"default_features\":false,\"features\":[\"std\"],\"name\":\"futures-io\",\"req\":\"^0.3.5\"},{\"default_features\":false,\"features\":[\"alloc\",\"async-await-macro\",\"std\"],\"name\":\"futures-util\",\"req\":\"^0.3.5\"},{\"features\":[\"stream\"],\"name\":\"h2\",\"optional\":true,\"req\":\"^0.4.0\"},{\"name\":\"h3\",\"optional\":true,\"req\":\"^0.0.8\"},{\"name\":\"h3-quinn\",\"optional\":true,\"req\":\"^0.0.10\"},{\"default_features\":false,\"features\":[\"std\"],\"name\":\"hickory-proto\",\"req\":\"^0.26.0\"},{\"name\":\"http\",\"optional\":true,\"req\":\"^1.1\"},{\"default_features\":false,\"features\":[\"alloc\",\"compiled_data\"],\"name\":\"idna\",\"req\":\"^1.0.3\"},{\"default_features\":false,\"name\":\"ipnet\",\"req\":\"^2.3.0\"},{\"name\":\"jni\",\"req\":\"^0.22.1\",\"target\":\"cfg(target_os = \\\"android\\\")\"},{\"name\":\"lru-cache\",\"optional\":true,\"req\":\"^0.1.2\"},{\"name\":\"parking_lot\",\"optional\":true,\"req\":\"^0.12\"},{\"name\":\"pin-project-lite\",\"optional\":true,\"req\":\"^0.2\"},{\"default_features\":false,\"features\":[\"log\",\"runtime-tokio\"],\"name\":\"quinn\",\"optional\":true,\"req\":\"^0.11.2\"},{\"default_features\":false,\"features\":[\"alloc\",\"std_rng\"],\"name\":\"rand\",\"req\":\"^0.10.0\"},{\"features\":[\"std\"],\"name\":\"ring\",\"optional\":true,\"req\":\"^0.17\"},{\"default_features\":false,\"features\":[\"logging\",\"std\",\"tls12\"],\"name\":\"rustls\",\"optional\":true,\"req\":\"^0.23.23\"},{\"name\":\"rustls-pki-types\",\"optional\":true,\"req\":\"^1.10\"},{\"name\":\"rustls-platform-verifier\",\"optional\":true,\"req\":\"^0.7\"},{\"features\":[\"derive\"],\"name\":\"serde\",\"optional\":true,\"req\":\"^1.0\"},{\"name\":\"socket2\",\"optional\":true,\"req\":\"^0.6\"},{\"default_features\":false,\"name\":\"thiserror\",\"req\":\"^2\"},{\"name\":\"time\",\"optional\":true,\"req\":\"^0.3\"},{\"features\":[\"alloc\"],\"name\":\"tinyvec\",\"req\":\"^1.1.1\"},{\"features\":[\"io-util\",\"macros\"],\"name\":\"tokio\",\"optional\":true,\"req\":\"^1.21\"},{\"features\":[\"rt\",\"time\",\"macros\",\"test-util\"],\"kind\":\"dev\",\"name\":\"tokio\",\"req\":\"^1.21\"},{\"default_features\":false,\"features\":[\"early-data\"],\"name\":\"tokio-rustls\",\"optional\":true,\"req\":\"^0.26\"},{\"default_features\":false,\"name\":\"tracing\",\"req\":\"^0.1.30\"},{\"default_features\":false,\"name\":\"url\",\"req\":\"^2.5.4\"},{\"name\":\"webpki-roots\",\"optional\":true,\"req\":\"^1\"}],\"features\":{\"__dnssec\":[\"dep:bitflags\",\"dep:rustls-pki-types\",\"dep:time\",\"dep:lru-cache\",\"dep:parking_lot\"],\"__h3\":[\"dep:h3\",\"dep:h3-quinn\",\"dep:http\"],\"__https\":[\"dep:bytes\",\"dep:h2\",\"dep:http\"],\"__quic\":[\"dep:bytes\",\"dep:pin-project-lite\",\"dep:quinn\"],\"__tls\":[\"dep:bytes\",\"dep:rustls\",\"dep:tokio-rustls\",\"tokio\"],\"default\":[\"tokio\"],\"dnssec-aws-lc-rs\":[\"dep:aws-lc-rs\",\"aws-lc-rs/aws-lc-sys\",\"aws-lc-rs/ring-io\",\"hickory-proto/dnssec-aws-lc-rs\",\"__dnssec\"],\"dnssec-ring\":[\"dep:ring\",\"hickory-proto/dnssec-ring\",\"__dnssec\"],\"h3-aws-lc-rs\":[\"quic-aws-lc-rs\",\"__h3\"],\"h3-ring\":[\"quic-ring\",\"__h3\"],\"https-aws-lc-rs\":[\"tls-aws-lc-rs\",\"__https\"],\"https-ring\":[\"tls-ring\",\"__https\"],\"mdns\":[\"socket2/all\"],\"quic-aws-lc-rs\":[\"quinn/rustls-aws-lc-rs\",\"tls-aws-lc-rs\",\"__quic\"],\"quic-ring\":[\"quinn/rustls-ring\",\"tls-ring\",\"__quic\"],\"rustls-platform-verifier\":[\"dep:rustls-platform-verifier\"],\"serde\":[\"dep:serde\",\"url/serde\"],\"tls-aws-lc-rs\":[\"tokio-rustls/aws-lc-rs\",\"__tls\"],\"tls-ring\":[\"tokio-rustls/ring\",\"__tls\"],\"tokio\":[\"dep:tokio\",\"tokio/net\",\"tokio/rt\",\"tokio/time\",\"tokio/rt-multi-thread\"]}}", "hickory-proto_0.26.1": "{\"dependencies\":[{\"default_features\":false,\"features\":[\"prebuilt-nasm\"],\"name\":\"aws-lc-rs\",\"optional\":true,\"req\":\"^1.12.3\"},{\"name\":\"bitflags\",\"optional\":true,\"req\":\"^2.4.1\"},{\"name\":\"critical-section\",\"optional\":true,\"req\":\"^1.1.1\"},{\"default_features\":false,\"features\":[\"alloc\"],\"name\":\"data-encoding\",\"req\":\"^2.2.0\"},{\"default_features\":false,\"features\":[\"alloc\",\"compiled_data\"],\"name\":\"idna\",\"req\":\"^1.0.3\"},{\"default_features\":false,\"name\":\"ipnet\",\"req\":\"^2.3.0\"},{\"name\":\"jni\",\"req\":\"^0.22.1\",\"target\":\"cfg(target_os = \\\"android\\\")\"},{\"name\":\"js-sys\",\"optional\":true,\"req\":\"^0.3.44\"},{\"default_features\":false,\"features\":[\"critical-section\"],\"name\":\"once_cell\",\"req\":\"^1.20.0\"},{\"name\":\"prefix-trie\",\"optional\":true,\"req\":\"^0.8\"},{\"default_features\":false,\"features\":[\"alloc\",\"std_rng\"],\"name\":\"rand\",\"req\":\"^0.10.0\"},{\"features\":[\"std\"],\"name\":\"ring\",\"optional\":true,\"req\":\"^0.17\"},{\"name\":\"rustls-pki-types\",\"optional\":true,\"req\":\"^1.10\"},{\"features\":[\"derive\"],\"name\":\"serde\",\"optional\":true,\"req\":\"^1.0\"},{\"default_features\":false,\"name\":\"thiserror\",\"req\":\"^2\"},{\"name\":\"time\",\"optional\":true,\"req\":\"^0.3\"},{\"features\":[\"alloc\"],\"name\":\"tinyvec\",\"req\":\"^1.1.1\"},{\"default_features\":false,\"name\":\"tracing\",\"req\":\"^0.1.30\"},{\"default_features\":false,\"name\":\"url\",\"req\":\"^2.5.4\"},{\"name\":\"wasm-bindgen-crate\",\"optional\":true,\"package\":\"wasm-bindgen\",\"req\":\"^0.2.58\"}],\"features\":{\"__dnssec\":[\"dep:bitflags\",\"dep:rustls-pki-types\",\"dep:time\",\"std\"],\"access-control\":[\"std\",\"dep:prefix-trie\"],\"default\":[\"std\"],\"dnssec-aws-lc-rs\":[\"dep:aws-lc-rs\",\"aws-lc-rs/aws-lc-sys\",\"aws-lc-rs/ring-io\",\"__dnssec\"],\"dnssec-ring\":[\"dep:ring\",\"__dnssec\"],\"mdns\":[],\"no-std-rand\":[\"once_cell/critical-section\",\"dep:critical-section\"],\"serde\":[\"dep:serde\",\"std\",\"url/serde\"],\"std\":[\"data-encoding/std\",\"ipnet/std\",\"rand/std\",\"rand/thread_rng\",\"ring?/std\",\"thiserror/std\",\"tracing/std\",\"url/std\"],\"testing\":[\"std\"],\"wasm-bindgen\":[\"dep:wasm-bindgen-crate\",\"dep:js-sys\"]}}", @@ -1021,7 +953,9 @@ "itoa_1.0.17": "{\"dependencies\":[{\"default_features\":false,\"kind\":\"dev\",\"name\":\"criterion\",\"req\":\"^0.8\",\"target\":\"cfg(not(miri))\"},{\"name\":\"no-panic\",\"optional\":true,\"req\":\"^0.1\"}],\"features\":{}}", "itoa_1.0.18": "{\"dependencies\":[{\"default_features\":false,\"kind\":\"dev\",\"name\":\"criterion\",\"req\":\"^0.8\",\"target\":\"cfg(not(miri))\"},{\"name\":\"no-panic\",\"optional\":true,\"req\":\"^0.1\"}],\"features\":{}}", "jiff-static_0.2.23": "{\"dependencies\":[{\"name\":\"jiff-tzdb\",\"optional\":true,\"req\":\"^0.1.6\"},{\"name\":\"proc-macro2\",\"req\":\"^1.0.93\"},{\"name\":\"quote\",\"req\":\"^1.0.38\"},{\"name\":\"syn\",\"req\":\"^2.0.98\"}],\"features\":{\"default\":[],\"perf-inline\":[],\"tz-fat\":[],\"tzdb\":[\"dep:jiff-tzdb\"]}}", + "jiff-static_0.2.24": "{\"dependencies\":[{\"name\":\"jiff-tzdb\",\"optional\":true,\"req\":\"^0.1.6\"},{\"name\":\"proc-macro2\",\"req\":\"^1.0.93\"},{\"name\":\"quote\",\"req\":\"^1.0.38\"},{\"name\":\"syn\",\"req\":\"^2.0.98\"}],\"features\":{\"default\":[],\"perf-inline\":[],\"tz-fat\":[],\"tzdb\":[\"dep:jiff-tzdb\"]}}", "jiff_0.2.23": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"anyhow\",\"req\":\"^1.0.81\"},{\"features\":[\"serde\"],\"kind\":\"dev\",\"name\":\"chrono\",\"req\":\"^0.4.38\"},{\"kind\":\"dev\",\"name\":\"chrono-tz\",\"req\":\"^0.10.0\"},{\"kind\":\"dev\",\"name\":\"hifitime\",\"req\":\"^3.9.0\",\"target\":\"cfg(not(target_family = \\\"wasm\\\"))\"},{\"kind\":\"dev\",\"name\":\"humantime\",\"req\":\"^2.1.0\"},{\"kind\":\"dev\",\"name\":\"insta\",\"req\":\"^1.39.0\"},{\"name\":\"jiff-static\",\"req\":\"=0.2.23\",\"target\":\"cfg(any())\"},{\"name\":\"jiff-static\",\"optional\":true,\"req\":\"^0.2\"},{\"name\":\"jiff-tzdb\",\"optional\":true,\"req\":\"^0.1.6\"},{\"name\":\"jiff-tzdb-platform\",\"optional\":true,\"req\":\"^0.1.3\",\"target\":\"cfg(any(windows, target_family = \\\"wasm\\\"))\"},{\"name\":\"js-sys\",\"optional\":true,\"req\":\"^0.3.50\",\"target\":\"cfg(all(any(target_arch = \\\"wasm32\\\", target_arch = \\\"wasm64\\\"), target_os = \\\"unknown\\\"))\"},{\"default_features\":false,\"name\":\"log\",\"optional\":true,\"req\":\"^0.4.21\"},{\"kind\":\"dev\",\"name\":\"log\",\"req\":\"^0.4.21\"},{\"default_features\":false,\"name\":\"portable-atomic\",\"req\":\"^1.10.0\",\"target\":\"cfg(not(target_has_atomic = \\\"ptr\\\"))\"},{\"default_features\":false,\"name\":\"portable-atomic-util\",\"req\":\"^0.2.4\",\"target\":\"cfg(not(target_has_atomic = \\\"ptr\\\"))\"},{\"default_features\":false,\"kind\":\"dev\",\"name\":\"quickcheck\",\"req\":\"^1.0.3\"},{\"features\":[\"derive\"],\"kind\":\"dev\",\"name\":\"serde\",\"req\":\"^1.0.203\"},{\"default_features\":false,\"name\":\"serde_core\",\"optional\":true,\"req\":\"^1.0.221\"},{\"kind\":\"dev\",\"name\":\"serde_json\",\"req\":\"^1.0.117\"},{\"kind\":\"dev\",\"name\":\"serde_yaml\",\"req\":\"^0.9.34\"},{\"kind\":\"dev\",\"name\":\"tabwriter\",\"req\":\"^1.4.0\"},{\"features\":[\"local-offset\",\"macros\",\"parsing\"],\"kind\":\"dev\",\"name\":\"time\",\"req\":\"^0.3.36\"},{\"kind\":\"dev\",\"name\":\"time-tz\",\"req\":\"^2.0.0\"},{\"kind\":\"dev\",\"name\":\"tzfile\",\"req\":\"^0.1.3\"},{\"kind\":\"dev\",\"name\":\"walkdir\",\"req\":\"^2.5.0\"},{\"name\":\"wasm-bindgen\",\"optional\":true,\"req\":\"^0.2.70\",\"target\":\"cfg(all(any(target_arch = \\\"wasm32\\\", target_arch = \\\"wasm64\\\"), target_os = \\\"unknown\\\"))\"},{\"default_features\":false,\"features\":[\"Win32_Foundation\",\"Win32_System_Time\"],\"name\":\"windows-sys\",\"optional\":true,\"req\":\">=0.52.0, <=0.61\",\"target\":\"cfg(windows)\"}],\"features\":{\"alloc\":[\"serde_core?/alloc\",\"portable-atomic-util/alloc\"],\"default\":[\"std\",\"tz-system\",\"tz-fat\",\"tzdb-bundle-platform\",\"tzdb-zoneinfo\",\"tzdb-concatenated\",\"perf-inline\"],\"js\":[\"dep:wasm-bindgen\",\"dep:js-sys\"],\"logging\":[\"dep:log\"],\"perf-inline\":[],\"serde\":[\"dep:serde_core\"],\"static\":[\"static-tz\",\"jiff-static?/tzdb\"],\"static-tz\":[\"dep:jiff-static\"],\"std\":[\"alloc\",\"log?/std\",\"serde_core?/std\"],\"tz-fat\":[\"jiff-static?/tz-fat\"],\"tz-system\":[\"std\",\"dep:windows-sys\"],\"tzdb-bundle-always\":[\"dep:jiff-tzdb\",\"alloc\"],\"tzdb-bundle-platform\":[\"dep:jiff-tzdb-platform\",\"alloc\"],\"tzdb-concatenated\":[\"std\"],\"tzdb-zoneinfo\":[\"std\"]}}", + "jiff_0.2.24": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"anyhow\",\"req\":\"^1.0.81\"},{\"features\":[\"serde\"],\"kind\":\"dev\",\"name\":\"chrono\",\"req\":\"^0.4.38\"},{\"kind\":\"dev\",\"name\":\"chrono-tz\",\"req\":\"^0.10.0\"},{\"kind\":\"dev\",\"name\":\"hifitime\",\"req\":\"^3.9.0\",\"target\":\"cfg(not(target_family = \\\"wasm\\\"))\"},{\"kind\":\"dev\",\"name\":\"humantime\",\"req\":\"^2.1.0\"},{\"kind\":\"dev\",\"name\":\"insta\",\"req\":\"^1.39.0\"},{\"name\":\"jiff-static\",\"req\":\"=0.2.24\",\"target\":\"cfg(any())\"},{\"name\":\"jiff-static\",\"optional\":true,\"req\":\"^0.2\"},{\"name\":\"jiff-tzdb\",\"optional\":true,\"req\":\"^0.1.6\"},{\"name\":\"jiff-tzdb-platform\",\"optional\":true,\"req\":\"^0.1.3\",\"target\":\"cfg(any(windows, target_family = \\\"wasm\\\"))\"},{\"name\":\"js-sys\",\"optional\":true,\"req\":\"^0.3.50\",\"target\":\"cfg(all(any(target_arch = \\\"wasm32\\\", target_arch = \\\"wasm64\\\"), target_os = \\\"unknown\\\"))\"},{\"default_features\":false,\"name\":\"log\",\"optional\":true,\"req\":\"^0.4.21\"},{\"kind\":\"dev\",\"name\":\"log\",\"req\":\"^0.4.21\"},{\"default_features\":false,\"name\":\"portable-atomic\",\"req\":\"^1.10.0\",\"target\":\"cfg(not(target_has_atomic = \\\"ptr\\\"))\"},{\"default_features\":false,\"name\":\"portable-atomic-util\",\"req\":\"^0.2.4\",\"target\":\"cfg(not(target_has_atomic = \\\"ptr\\\"))\"},{\"default_features\":false,\"kind\":\"dev\",\"name\":\"quickcheck\",\"req\":\"^1.0.3\"},{\"features\":[\"derive\"],\"kind\":\"dev\",\"name\":\"serde\",\"req\":\"^1.0.203\"},{\"default_features\":false,\"name\":\"serde_core\",\"optional\":true,\"req\":\"^1.0.221\"},{\"kind\":\"dev\",\"name\":\"serde_json\",\"req\":\"^1.0.117\"},{\"kind\":\"dev\",\"name\":\"serde_yaml\",\"req\":\"^0.9.34\"},{\"kind\":\"dev\",\"name\":\"tabwriter\",\"req\":\"^1.4.0\"},{\"features\":[\"local-offset\",\"macros\",\"parsing\"],\"kind\":\"dev\",\"name\":\"time\",\"req\":\"^0.3.36\"},{\"kind\":\"dev\",\"name\":\"time-tz\",\"req\":\"^2.0.0\"},{\"kind\":\"dev\",\"name\":\"tzfile\",\"req\":\"^0.1.3\"},{\"kind\":\"dev\",\"name\":\"walkdir\",\"req\":\"^2.5.0\"},{\"name\":\"wasm-bindgen\",\"optional\":true,\"req\":\"^0.2.70\",\"target\":\"cfg(all(any(target_arch = \\\"wasm32\\\", target_arch = \\\"wasm64\\\"), target_os = \\\"unknown\\\"))\"},{\"default_features\":false,\"features\":[\"Win32_Foundation\",\"Win32_System_Time\"],\"name\":\"windows-sys\",\"optional\":true,\"req\":\">=0.52.0, <=0.61\",\"target\":\"cfg(windows)\"}],\"features\":{\"alloc\":[\"serde_core?/alloc\",\"portable-atomic-util/alloc\"],\"default\":[\"std\",\"tz-system\",\"tz-fat\",\"tzdb-bundle-platform\",\"tzdb-zoneinfo\",\"tzdb-concatenated\",\"perf-inline\"],\"js\":[\"dep:wasm-bindgen\",\"dep:js-sys\"],\"logging\":[\"dep:log\"],\"perf-inline\":[],\"serde\":[\"dep:serde_core\"],\"static\":[\"static-tz\",\"jiff-static?/tzdb\"],\"static-tz\":[\"dep:jiff-static\"],\"std\":[\"alloc\",\"log?/std\",\"serde_core?/std\"],\"tz-fat\":[\"jiff-static?/tz-fat\"],\"tz-system\":[\"std\",\"dep:windows-sys\"],\"tzdb-bundle-always\":[\"dep:jiff-tzdb\",\"alloc\"],\"tzdb-bundle-platform\":[\"dep:jiff-tzdb-platform\",\"alloc\"],\"tzdb-concatenated\":[\"std\"],\"tzdb-zoneinfo\":[\"std\"]}}", "jni-macros_0.22.4": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"javac\",\"req\":\"^0.1.0\"},{\"name\":\"proc-macro2\",\"req\":\"^1.0\"},{\"name\":\"quote\",\"req\":\"^1.0\"},{\"kind\":\"build\",\"name\":\"rustc_version\",\"req\":\"^0.4\"},{\"kind\":\"dev\",\"name\":\"rusty-fork\",\"req\":\"^0.3.0\"},{\"name\":\"simd_cesu8\",\"req\":\"^1.0.1\"},{\"features\":[\"full\"],\"name\":\"syn\",\"req\":\"^2.0\"},{\"kind\":\"dev\",\"name\":\"thiserror\",\"req\":\"^2\"},{\"kind\":\"dev\",\"name\":\"trybuild\",\"req\":\"^1\"}],\"features\":{}}", "jni-sys-macros_0.4.1": "{\"dependencies\":[{\"name\":\"quote\",\"req\":\"^1\"},{\"features\":[\"full\"],\"name\":\"syn\",\"req\":\"^2\"}],\"features\":{}}", "jni-sys_0.3.0": "{\"dependencies\":[],\"features\":{}}", @@ -1030,8 +964,10 @@ "jni_0.22.4": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"assert_matches\",\"req\":\"^1.5.0\"},{\"kind\":\"dev\",\"name\":\"bytemuck\",\"req\":\"^1.13.0\",\"target\":\"cfg(windows)\"},{\"name\":\"cfg-if\",\"req\":\"^1.0.0\"},{\"name\":\"combine\",\"req\":\"^4.1.0\"},{\"features\":[\"html_reports\"],\"kind\":\"dev\",\"name\":\"criterion\",\"req\":\"^0.7\"},{\"name\":\"java-locator\",\"optional\":true,\"req\":\"^0.1.3\",\"target\":\"cfg(not(target_os = \\\"android\\\"))\"},{\"kind\":\"dev\",\"name\":\"javac\",\"req\":\"^0.1.0\"},{\"name\":\"jni-macros\",\"req\":\"=0.22.4\"},{\"name\":\"jni-sys\",\"req\":\"^0.4.1\"},{\"kind\":\"dev\",\"name\":\"lazy_static\",\"req\":\"^1\"},{\"name\":\"libloading\",\"optional\":true,\"req\":\"^0.8\",\"target\":\"cfg(not(target_os = \\\"android\\\"))\"},{\"name\":\"log\",\"req\":\"^0.4.4\"},{\"kind\":\"dev\",\"name\":\"rusty-fork\",\"req\":\"^0.3.0\"},{\"name\":\"simd_cesu8\",\"req\":\"^1.1.1\"},{\"kind\":\"dev\",\"name\":\"static_assertions\",\"req\":\"^1\"},{\"name\":\"thiserror\",\"req\":\"^2\"},{\"kind\":\"dev\",\"name\":\"trybuild\",\"req\":\"^1\"},{\"kind\":\"build\",\"name\":\"walkdir\",\"req\":\"^2\"},{\"name\":\"windows-link\",\"req\":\"^0.2\",\"target\":\"cfg(windows)\"},{\"features\":[\"Win32_System_Threading\",\"Win32_Foundation\"],\"kind\":\"dev\",\"name\":\"windows-sys\",\"req\":\"^0.61\",\"target\":\"cfg(windows)\"}],\"features\":{\"_cfg_test\":[],\"default\":[],\"invocation\":[\"dep:java-locator\",\"dep:libloading\"]}}", "jobserver_0.1.34": "{\"dependencies\":[{\"features\":[\"std\"],\"name\":\"getrandom\",\"req\":\"^0.3.2\",\"target\":\"cfg(windows)\"},{\"name\":\"libc\",\"req\":\"^0.2.171\",\"target\":\"cfg(unix)\"},{\"features\":[\"fs\"],\"kind\":\"dev\",\"name\":\"nix\",\"req\":\"^0.28.0\",\"target\":\"cfg(unix)\"},{\"kind\":\"dev\",\"name\":\"tempfile\",\"req\":\"^3.10.1\"}],\"features\":{}}", "js-sys_0.3.102": "{\"dependencies\":[{\"name\":\"cfg-if\",\"req\":\"^1.0.0\"},{\"kind\":\"dev\",\"name\":\"futures-channel\",\"req\":\"^0.3\",\"target\":\"cfg(target_arch = \\\"wasm32\\\")\"},{\"default_features\":false,\"name\":\"futures-core\",\"optional\":true,\"req\":\"^0.3.8\"},{\"default_features\":false,\"kind\":\"dev\",\"name\":\"futures-lite\",\"req\":\"^2\",\"target\":\"cfg(target_arch = \\\"wasm32\\\")\"},{\"default_features\":false,\"features\":[\"std\"],\"name\":\"futures-util\",\"optional\":true,\"req\":\"^0.3.31\"},{\"kind\":\"dev\",\"name\":\"half\",\"req\":\"^2\",\"target\":\"cfg(target_arch = \\\"wasm32\\\")\"},{\"default_features\":false,\"name\":\"wasm-bindgen\",\"req\":\"=0.2.125\"}],\"features\":{\"default\":[\"std\",\"unsafe-eval\"],\"futures-core-03-stream\":[\"dep:futures-util\",\"dep:futures-core\"],\"std\":[\"wasm-bindgen/std\",\"dep:futures-util\"],\"unsafe-eval\":[]}}", + "js-sys_0.3.82": "{\"dependencies\":[{\"default_features\":false,\"name\":\"once_cell\",\"req\":\"^1.12\"},{\"default_features\":false,\"name\":\"wasm-bindgen\",\"req\":\"=0.2.105\"}],\"features\":{\"default\":[\"std\"],\"std\":[\"wasm-bindgen/std\"]}}", "jsonwebtoken_10.3.0": "{\"dependencies\":[{\"name\":\"aws-lc-rs\",\"optional\":true,\"req\":\"^1.15.0\"},{\"name\":\"base64\",\"req\":\"^0.22\"},{\"default_features\":false,\"kind\":\"dev\",\"name\":\"criterion\",\"req\":\"^0.8\",\"target\":\"cfg(all(target_arch = \\\"wasm32\\\", not(any(target_os = \\\"emscripten\\\", target_os = \\\"wasi\\\"))))\"},{\"kind\":\"dev\",\"name\":\"criterion\",\"req\":\"^0.8\",\"target\":\"cfg(not(all(target_arch = \\\"wasm32\\\", not(any(target_os = \\\"emscripten\\\", target_os = \\\"wasi\\\")))))\"},{\"features\":[\"pkcs8\"],\"name\":\"ed25519-dalek\",\"optional\":true,\"req\":\"^2.1.1\"},{\"features\":[\"pkcs8\",\"rand_core\"],\"kind\":\"dev\",\"name\":\"ed25519-dalek\",\"req\":\"^2.1.1\"},{\"name\":\"getrandom\",\"req\":\"^0.2\",\"target\":\"cfg(target_arch = \\\"wasm32\\\")\"},{\"name\":\"hmac\",\"optional\":true,\"req\":\"^0.12.1\"},{\"name\":\"js-sys\",\"req\":\"^0.3\",\"target\":\"cfg(target_arch = \\\"wasm32\\\")\"},{\"features\":[\"ecdsa\"],\"name\":\"p256\",\"optional\":true,\"req\":\"^0.13.2\"},{\"features\":[\"ecdsa\"],\"name\":\"p384\",\"optional\":true,\"req\":\"^0.13.0\"},{\"name\":\"pem\",\"optional\":true,\"req\":\"^3\"},{\"default_features\":false,\"features\":[\"std\"],\"name\":\"rand\",\"optional\":true,\"req\":\"^0.8.5\"},{\"default_features\":false,\"features\":[\"std\"],\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.8.5\"},{\"kind\":\"dev\",\"name\":\"rand_core\",\"req\":\"^0.6.4\"},{\"name\":\"rsa\",\"optional\":true,\"req\":\"^0.9.6\"},{\"features\":[\"derive\"],\"name\":\"serde\",\"req\":\"^1.0.228\"},{\"name\":\"serde_json\",\"req\":\"^1.0\"},{\"features\":[\"oid\"],\"name\":\"sha2\",\"optional\":true,\"req\":\"^0.10.7\"},{\"features\":[\"std\"],\"name\":\"signature\",\"req\":\"^2.2.0\"},{\"name\":\"simple_asn1\",\"optional\":true,\"req\":\"^0.6\"},{\"features\":[\"wasm-bindgen\"],\"kind\":\"dev\",\"name\":\"time\",\"req\":\"^0.3\",\"target\":\"cfg(all(target_arch = \\\"wasm32\\\", not(any(target_os = \\\"emscripten\\\", target_os = \\\"wasi\\\"))))\"},{\"kind\":\"dev\",\"name\":\"time\",\"req\":\"^0.3\",\"target\":\"cfg(not(all(target_arch = \\\"wasm32\\\", not(any(target_os = \\\"emscripten\\\", target_os = \\\"wasi\\\")))))\"},{\"kind\":\"dev\",\"name\":\"wasm-bindgen-test\",\"req\":\"^0.3.1\"}],\"features\":{\"aws_lc_rs\":[\"aws-lc-rs\"],\"default\":[\"use_pem\"],\"rust_crypto\":[\"ed25519-dalek\",\"hmac\",\"p256\",\"p384\",\"rand\",\"rsa\",\"sha2\"],\"use_pem\":[\"pem\",\"simple_asn1\"]}}", "lazy_static_1.5.0": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"doc-comment\",\"req\":\"^0.3.1\"},{\"default_features\":false,\"features\":[\"once\"],\"name\":\"spin\",\"optional\":true,\"req\":\"^0.9.8\"},{\"kind\":\"dev\",\"name\":\"trybuild\",\"req\":\"^1\"}],\"features\":{\"spin_no_std\":[\"spin\"]}}", + "leb128_0.2.6": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"quickcheck\",\"req\":\"^1.1\"}],\"features\":{\"nightly\":[]}}", "leb128fmt_0.1.0": "{\"dependencies\":[],\"features\":{\"alloc\":[],\"default\":[\"std\"],\"std\":[]}}", "libc_0.2.183": "{\"dependencies\":[{\"name\":\"rustc-std-workspace-core\",\"optional\":true,\"req\":\"^1.0.1\"}],\"features\":{\"align\":[],\"const-extern-fn\":[],\"default\":[\"std\"],\"extra_traits\":[],\"rustc-dep-of-std\":[\"align\",\"rustc-std-workspace-core\"],\"std\":[],\"use_std\":[\"std\"]}}", "libc_0.2.186": "{\"dependencies\":[{\"name\":\"rustc-std-workspace-core\",\"optional\":true,\"req\":\"^1.0.1\"}],\"features\":{\"align\":[],\"const-extern-fn\":[],\"default\":[\"std\"],\"extra_traits\":[],\"rustc-dep-of-std\":[\"align\",\"rustc-std-workspace-core\"],\"std\":[],\"use_std\":[\"std\"]}}", @@ -1060,6 +996,7 @@ "mimalloc_0.1.48": "{\"dependencies\":[{\"default_features\":false,\"name\":\"libmimalloc-sys\",\"req\":\"^0.1.44\"}],\"features\":{\"debug\":[\"libmimalloc-sys/debug\"],\"debug_in_debug\":[\"libmimalloc-sys/debug_in_debug\"],\"default\":[],\"extended\":[\"libmimalloc-sys/extended\"],\"local_dynamic_tls\":[\"libmimalloc-sys/local_dynamic_tls\"],\"no_thp\":[\"libmimalloc-sys/no_thp\"],\"override\":[\"libmimalloc-sys/override\"],\"secure\":[\"libmimalloc-sys/secure\"],\"v3\":[\"libmimalloc-sys/v3\"]}}", "mime_0.3.17": "{\"dependencies\":[],\"features\":{}}", "mime_guess_2.0.5": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"criterion\",\"req\":\"^0.3\"},{\"name\":\"mime\",\"req\":\"^0.3\"},{\"name\":\"unicase\",\"req\":\"^2.4.0\"},{\"kind\":\"build\",\"name\":\"unicase\",\"req\":\"^2.4.0\"}],\"features\":{\"default\":[\"rev-mappings\"],\"rev-mappings\":[]}}", + "minicov_0.3.8": "{\"dependencies\":[{\"kind\":\"build\",\"name\":\"cc\",\"req\":\"^1.0.77\"},{\"kind\":\"build\",\"name\":\"walkdir\",\"req\":\"^2.3.2\"}],\"features\":{\"alloc\":[],\"default\":[\"alloc\"]}}", "minimal-lexical_0.2.1": "{\"dependencies\":[],\"features\":{\"alloc\":[],\"compact\":[],\"default\":[\"std\"],\"lint\":[],\"nightly\":[],\"std\":[]}}", "miniz_oxide_0.8.9": "{\"dependencies\":[{\"default_features\":false,\"name\":\"adler2\",\"req\":\"^2.0\"},{\"name\":\"alloc\",\"optional\":true,\"package\":\"rustc-std-workspace-alloc\",\"req\":\"^1.0.0\"},{\"name\":\"core\",\"optional\":true,\"package\":\"rustc-std-workspace-core\",\"req\":\"^1.0.0\"},{\"features\":[\"derive\"],\"name\":\"serde\",\"optional\":true,\"req\":\"^1.0\"},{\"default_features\":false,\"name\":\"simd-adler32\",\"optional\":true,\"req\":\"^0.3.3\"}],\"features\":{\"block-boundary\":[],\"default\":[\"with-alloc\"],\"rustc-dep-of-std\":[\"core\",\"alloc\",\"adler2/rustc-dep-of-std\"],\"simd\":[\"simd-adler32\"],\"std\":[],\"with-alloc\":[]}}", "mio_1.1.1": "{\"dependencies\":[{\"default_features\":false,\"kind\":\"dev\",\"name\":\"env_logger\",\"req\":\"^0.11\"},{\"name\":\"libc\",\"req\":\"^0.2.178\",\"target\":\"cfg(target_os = \\\"hermit\\\")\"},{\"name\":\"libc\",\"req\":\"^0.2.178\",\"target\":\"cfg(target_os = \\\"wasi\\\")\"},{\"name\":\"libc\",\"req\":\"^0.2.178\",\"target\":\"cfg(unix)\"},{\"name\":\"log\",\"optional\":true,\"req\":\"^0.4.8\"},{\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.9\"},{\"name\":\"wasi\",\"req\":\"^0.11.0\",\"target\":\"cfg(target_os = \\\"wasi\\\")\"},{\"features\":[\"Wdk_Foundation\",\"Wdk_Storage_FileSystem\",\"Wdk_System_IO\",\"Win32_Foundation\",\"Win32_Networking_WinSock\",\"Win32_Storage_FileSystem\",\"Win32_Security\",\"Win32_System_IO\",\"Win32_System_WindowsProgramming\"],\"name\":\"windows-sys\",\"req\":\"^0.61\",\"target\":\"cfg(windows)\"}],\"features\":{\"default\":[\"log\"],\"net\":[],\"os-ext\":[\"os-poll\",\"windows-sys/Win32_System_Pipes\",\"windows-sys/Win32_Security\"],\"os-poll\":[]}}", @@ -1071,16 +1008,20 @@ "mongodb-internal-macros_3.3.0": "{\"dependencies\":[{\"features\":[\"proc_support\"],\"name\":\"macro_magic\",\"req\":\"^0.5.1\"},{\"name\":\"proc-macro2\",\"req\":\"^1.0.78\"},{\"name\":\"quote\",\"req\":\"^1.0.35\"},{\"features\":[\"full\",\"parsing\",\"proc-macro\",\"extra-traits\"],\"name\":\"syn\",\"req\":\"^2.0.52\"}],\"features\":{}}", "mongodb_3.3.0": "{\"dependencies\":[{\"features\":[\"backtrace\"],\"kind\":\"dev\",\"name\":\"anyhow\",\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"approx\",\"req\":\"^0.5.1\"},{\"name\":\"async-trait\",\"req\":\"^0.1.42\"},{\"default_features\":false,\"features\":[\"default-https-client\",\"rt-tokio\"],\"name\":\"aws-config\",\"optional\":true,\"req\":\"^1\"},{\"default_features\":false,\"name\":\"aws-credential-types\",\"optional\":true,\"req\":\"^1.2.4\"},{\"default_features\":false,\"features\":[\"sign-http\"],\"name\":\"aws-sigv4\",\"optional\":true,\"req\":\"^1.3.3\"},{\"default_features\":false,\"name\":\"aws-types\",\"optional\":true,\"req\":\"^1.3.7\"},{\"kind\":\"dev\",\"name\":\"backtrace\",\"req\":\"^0.3.68\"},{\"name\":\"base64\",\"req\":\"^0.13.0\"},{\"name\":\"bitflags\",\"req\":\"^1.1.0\"},{\"name\":\"bson2\",\"optional\":true,\"package\":\"bson\",\"req\":\"^2.15.0\"},{\"features\":[\"serde\"],\"name\":\"bson3\",\"optional\":true,\"package\":\"bson\",\"req\":\"^3.0.0\"},{\"features\":[\"serde\",\"serde_json-1\"],\"kind\":\"dev\",\"name\":\"bson3\",\"package\":\"bson\",\"req\":\"^3.0.0\"},{\"default_features\":false,\"features\":[\"clock\",\"std\"],\"name\":\"chrono\",\"req\":\"^0.4.7\"},{\"default_features\":false,\"name\":\"cross-krb5\",\"optional\":true,\"req\":\"^0.4.2\",\"target\":\"cfg(not(windows))\"},{\"kind\":\"dev\",\"name\":\"ctrlc\",\"req\":\"^3.2.2\"},{\"name\":\"derive-where\",\"req\":\"^1.2.7\"},{\"name\":\"derive_more\",\"req\":\"^0.99.17\"},{\"name\":\"flate2\",\"optional\":true,\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"function_name\",\"req\":\"^0.2.1\"},{\"kind\":\"dev\",\"name\":\"futures\",\"req\":\"^0.3\"},{\"name\":\"futures-core\",\"req\":\"^0.3.14\"},{\"name\":\"futures-executor\",\"req\":\"^0.3.14\"},{\"name\":\"futures-io\",\"req\":\"^0.3.21\"},{\"features\":[\"io\"],\"name\":\"futures-util\",\"req\":\"^0.3.14\"},{\"name\":\"hex\",\"req\":\"^0.4.0\"},{\"kind\":\"dev\",\"name\":\"hex\",\"req\":\"^0.4\"},{\"name\":\"hickory-proto\",\"optional\":true,\"req\":\"^0.24.2\"},{\"name\":\"hickory-resolver\",\"optional\":true,\"req\":\"^0.24.2\"},{\"name\":\"hmac\",\"req\":\"^0.12.1\"},{\"kind\":\"dev\",\"name\":\"home\",\"req\":\"^0.5\"},{\"default_features\":false,\"name\":\"http\",\"optional\":true,\"req\":\"^1.3\"},{\"kind\":\"dev\",\"name\":\"lambda_runtime\",\"req\":\"^0.6.0\"},{\"name\":\"log\",\"optional\":true,\"req\":\"^0.4.17\"},{\"name\":\"macro_magic\",\"req\":\"^0.5.1\"},{\"name\":\"md-5\",\"req\":\"^0.10.1\"},{\"default_features\":false,\"name\":\"mongocrypt\",\"optional\":true,\"req\":\"^0.3.1\"},{\"name\":\"mongodb-internal-macros\",\"req\":\"^3.3.0\"},{\"name\":\"num_cpus\",\"optional\":true,\"req\":\"^1.13.1\"},{\"name\":\"once_cell\",\"req\":\"^1.19.0\"},{\"name\":\"openssl\",\"optional\":true,\"req\":\"^0.10.38\"},{\"name\":\"openssl-probe\",\"optional\":true,\"req\":\"^0.1.5\"},{\"default_features\":false,\"name\":\"pbkdf2\",\"req\":\"^0.11.0\"},{\"name\":\"pem\",\"optional\":true,\"req\":\"^3.0.4\"},{\"name\":\"percent-encoding\",\"req\":\"^2.0.0\"},{\"features\":[\"encryption\",\"pkcs5\"],\"name\":\"pkcs8\",\"optional\":true,\"req\":\"^0.10.2\"},{\"features\":[\"3des\",\"des-insecure\",\"sha1-insecure\"],\"kind\":\"dev\",\"name\":\"pkcs8\",\"req\":\"^0.10.2\"},{\"kind\":\"dev\",\"name\":\"pretty_assertions\",\"req\":\"^1.3.0\"},{\"features\":[\"small_rng\"],\"name\":\"rand\",\"req\":\"^0.8.3\"},{\"name\":\"rayon\",\"optional\":true,\"req\":\"^1.5.3\"},{\"kind\":\"dev\",\"name\":\"regex\",\"req\":\"^1.6.0\"},{\"default_features\":false,\"features\":[\"json\",\"rustls-tls\"],\"name\":\"reqwest\",\"optional\":true,\"req\":\"^0.12.12\"},{\"features\":[\"rustls-tls\"],\"kind\":\"dev\",\"name\":\"reqwest\",\"req\":\"^0.12.2\"},{\"name\":\"rustc_version_runtime\",\"req\":\"^0.3.0\"},{\"default_features\":false,\"features\":[\"logging\",\"ring\",\"std\",\"tls12\"],\"name\":\"rustls\",\"optional\":true,\"req\":\"^0.23.20\"},{\"name\":\"rustversion\",\"req\":\"^1.0.20\"},{\"kind\":\"dev\",\"name\":\"semver\",\"req\":\"^1.0.0\"},{\"features\":[\"derive\"],\"name\":\"serde\",\"req\":\"^1.0.125\"},{\"features\":[\"rc\"],\"kind\":\"dev\",\"name\":\"serde\",\"req\":\">=0.0.0\"},{\"kind\":\"dev\",\"name\":\"serde-hex\",\"req\":\"^0.1.0\"},{\"name\":\"serde_bytes\",\"req\":\"^0.11.5\"},{\"kind\":\"dev\",\"name\":\"serde_json\",\"req\":\"^1.0.64\"},{\"kind\":\"dev\",\"name\":\"serde_path_to_error\",\"req\":\"^0.1\"},{\"name\":\"serde_with\",\"req\":\"^3.8.1\"},{\"name\":\"sha1\",\"req\":\"^0.10.0\"},{\"name\":\"sha2\",\"req\":\"^0.10.2\"},{\"name\":\"snap\",\"optional\":true,\"req\":\"^1.0.5\"},{\"name\":\"socket2\",\"req\":\"^0.5.5\"},{\"name\":\"stringprep\",\"req\":\"^0.1.2\"},{\"name\":\"strsim\",\"req\":\"^0.11.1\"},{\"name\":\"take_mut\",\"req\":\"^0.2.2\"},{\"name\":\"thiserror\",\"req\":\"^1.0.24\"},{\"kind\":\"dev\",\"name\":\"time\",\"req\":\"^0.3.9\"},{\"features\":[\"io-util\",\"sync\",\"macros\",\"net\",\"process\",\"rt\",\"time\",\"fs\"],\"name\":\"tokio\",\"req\":\"^1.17.0\"},{\"features\":[\"fs\",\"parking_lot\"],\"kind\":\"dev\",\"name\":\"tokio\",\"req\":\">=0.0.0\"},{\"name\":\"tokio-openssl\",\"optional\":true,\"req\":\"^0.6.3\"},{\"default_features\":false,\"features\":[\"logging\",\"ring\",\"tls12\"],\"name\":\"tokio-rustls\",\"optional\":true,\"req\":\"^0.26\"},{\"features\":[\"io\",\"compat\"],\"name\":\"tokio-util\",\"req\":\"^0.7.0\"},{\"name\":\"tracing\",\"optional\":true,\"req\":\"^0.1.36\"},{\"kind\":\"dev\",\"name\":\"tracing-subscriber\",\"req\":\"^0.3.16\"},{\"name\":\"typed-builder\",\"req\":\"^0.20.0\"},{\"features\":[\"v4\"],\"name\":\"uuid\",\"req\":\"^1.1.2\"},{\"name\":\"webpki-roots\",\"req\":\"^0.26\"},{\"features\":[\"Win32_Security_Authentication_Identity\",\"Win32_Security_Credentials\",\"Win32_Foundation\",\"Win32_System\",\"Win32_System_Rpc\"],\"name\":\"windows-sys\",\"optional\":true,\"req\":\"^0.60\",\"target\":\"cfg(windows)\"},{\"name\":\"zstd\",\"optional\":true,\"req\":\"^0.11.2\"}],\"features\":{\"aws-auth\":[\"dep:aws-config\",\"dep:aws-types\",\"dep:aws-credential-types\",\"dep:aws-sigv4\",\"dep:http\"],\"azure-kms\":[\"dep:reqwest\"],\"azure-oidc\":[\"dep:reqwest\"],\"bson-2\":[\"dep:bson2\",\"mongocrypt/bson-2\"],\"bson-3\":[\"dep:bson3\",\"mongocrypt/bson-3\"],\"cert-key-password\":[\"dep:pem\",\"dep:pkcs8\"],\"compat-3-0-0\":[\"compat-3-3-0\",\"bson-2\"],\"compat-3-3-0\":[],\"default\":[\"compat-3-0-0\",\"rustls-tls\",\"dns-resolver\"],\"dns-resolver\":[\"dep:hickory-resolver\",\"dep:hickory-proto\"],\"gcp-kms\":[\"dep:reqwest\"],\"gcp-oidc\":[\"dep:reqwest\"],\"gssapi-auth\":[\"dep:cross-krb5\",\"dep:windows-sys\",\"dns-resolver\"],\"in-use-encryption\":[\"dep:mongocrypt\",\"dep:rayon\",\"dep:num_cpus\"],\"in-use-encryption-unstable\":[\"in-use-encryption\"],\"openssl-tls\":[\"dep:openssl\",\"dep:openssl-probe\",\"dep:tokio-openssl\"],\"rustls-tls\":[\"dep:rustls\",\"dep:tokio-rustls\"],\"snappy-compression\":[\"dep:snap\"],\"sync\":[],\"tracing-unstable\":[\"dep:tracing\",\"dep:log\",\"bson3?/serde_json-1\"],\"zlib-compression\":[\"dep:flate2\"],\"zstd-compression\":[\"dep:zstd\"]}}", "multimap_0.10.1": "{\"dependencies\":[{\"name\":\"serde\",\"optional\":true,\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"serde_test\",\"req\":\"^1.0\"}],\"features\":{\"default\":[\"serde_impl\"],\"serde_impl\":[\"serde\"]}}", + "multipart_0.18.0": "{\"dependencies\":[{\"default_features\":false,\"name\":\"buf_redux\",\"optional\":true,\"req\":\"^0.8\"},{\"name\":\"clippy\",\"optional\":true,\"req\":\">=0.0, <0.1\"},{\"kind\":\"dev\",\"name\":\"env_logger\",\"req\":\"^0.5\"},{\"name\":\"httparse\",\"optional\":true,\"req\":\"^1.2\"},{\"default_features\":false,\"name\":\"hyper\",\"optional\":true,\"req\":\">=0.9, <0.11\"},{\"name\":\"iron\",\"optional\":true,\"req\":\">=0.4, <0.7\"},{\"name\":\"lazy_static\",\"optional\":true,\"req\":\"^1.2.0\"},{\"name\":\"log\",\"req\":\"^0.4\"},{\"name\":\"mime\",\"req\":\"^0.3.14\"},{\"name\":\"mime_guess\",\"req\":\"^2.0.1\"},{\"name\":\"nickel\",\"optional\":true,\"req\":\">=0.10.1\"},{\"name\":\"quick-error\",\"optional\":true,\"req\":\"^1.2\"},{\"name\":\"rand\",\"req\":\"^0.8\"},{\"name\":\"rocket\",\"optional\":true,\"req\":\"^0.4\"},{\"name\":\"safemem\",\"optional\":true,\"req\":\"^0.3\"},{\"name\":\"tempfile\",\"req\":\"^3\"},{\"name\":\"tiny_http\",\"optional\":true,\"req\":\"^0.6\"},{\"name\":\"twoway\",\"optional\":true,\"req\":\"^0.1\"}],\"features\":{\"bench\":[],\"client\":[],\"default\":[\"client\",\"hyper\",\"iron\",\"mock\",\"nickel\",\"server\",\"tiny_http\"],\"mock\":[],\"nightly\":[],\"server\":[\"buf_redux\",\"httparse\",\"quick-error\",\"safemem\",\"twoway\"]}}", "ndk-context_0.1.1": "{\"dependencies\":[],\"features\":{}}", "nom_7.1.3": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"doc-comment\",\"req\":\"^0.3\"},{\"default_features\":false,\"name\":\"memchr\",\"req\":\"^2.3\"},{\"default_features\":false,\"name\":\"minimal-lexical\",\"req\":\"^0.2.0\"},{\"kind\":\"dev\",\"name\":\"proptest\",\"req\":\"^1.0.0\"}],\"features\":{\"alloc\":[],\"default\":[\"std\"],\"docsrs\":[],\"std\":[\"alloc\",\"memchr/std\",\"minimal-lexical/std\"]}}", "nu-ansi-term_0.50.3": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"doc-comment\",\"req\":\"^0.3.3\"},{\"features\":[\"derive\"],\"name\":\"serde\",\"optional\":true,\"req\":\"^1.0.152\"},{\"kind\":\"dev\",\"name\":\"serde_json\",\"req\":\"^1.0.94\"},{\"features\":[\"Win32_Foundation\",\"Win32_System_Console\",\"Win32_Storage_FileSystem\",\"Win32_Security\"],\"name\":\"windows\",\"package\":\"windows-sys\",\"req\":\">=0.59, <=0.61\",\"target\":\"cfg(windows)\"}],\"features\":{\"default\":[\"std\"],\"derive_serde_style\":[\"serde\"],\"gnu_legacy\":[],\"std\":[]}}", "num-bigint-dig_0.8.6": "{\"dependencies\":[{\"name\":\"arbitrary\",\"optional\":true,\"req\":\"^1.1.0\"},{\"default_features\":false,\"features\":[\"spin_no_std\"],\"name\":\"lazy_static\",\"req\":\"^1.2.0\"},{\"name\":\"libm\",\"req\":\"^0.2.1\"},{\"default_features\":false,\"features\":[\"i128\"],\"name\":\"num-integer\",\"req\":\"^0.1.39\"},{\"default_features\":false,\"name\":\"num-iter\",\"req\":\"^0.1.37\"},{\"default_features\":false,\"features\":[\"i128\"],\"name\":\"num-traits\",\"req\":\"^0.2.4\"},{\"default_features\":false,\"name\":\"rand\",\"optional\":true,\"req\":\"^0.8.3\"},{\"features\":[\"small_rng\"],\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.8\"},{\"kind\":\"dev\",\"name\":\"rand_chacha\",\"req\":\"^0.3\"},{\"kind\":\"dev\",\"name\":\"rand_isaac\",\"req\":\"^0.3\"},{\"kind\":\"dev\",\"name\":\"rand_xorshift\",\"req\":\"^0.3\"},{\"default_features\":false,\"features\":[\"alloc\"],\"name\":\"serde\",\"optional\":true,\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"serde_test\",\"req\":\"^1.0\"},{\"default_features\":false,\"name\":\"smallvec\",\"req\":\"^1.10.0\"},{\"default_features\":false,\"name\":\"zeroize\",\"optional\":true,\"req\":\"^1.5\"}],\"features\":{\"default\":[\"std\",\"u64_digit\"],\"fuzz\":[\"arbitrary\",\"smallvec/arbitrary\"],\"i128\":[],\"nightly\":[],\"prime\":[\"rand/std_rng\"],\"std\":[\"num-integer/std\",\"num-traits/std\",\"smallvec/write\",\"rand/std\",\"serde/std\"],\"u64_digit\":[]}}", "num-bigint_0.4.6": "{\"dependencies\":[{\"default_features\":false,\"name\":\"arbitrary\",\"optional\":true,\"req\":\"^1\"},{\"default_features\":false,\"features\":[\"i128\"],\"name\":\"num-integer\",\"req\":\"^0.1.46\"},{\"default_features\":false,\"features\":[\"i128\"],\"name\":\"num-traits\",\"req\":\"^0.2.18\"},{\"default_features\":false,\"name\":\"quickcheck\",\"optional\":true,\"req\":\"^1\"},{\"default_features\":false,\"name\":\"rand\",\"optional\":true,\"req\":\"^0.8\"},{\"default_features\":false,\"name\":\"serde\",\"optional\":true,\"req\":\"^1.0\"}],\"features\":{\"arbitrary\":[\"dep:arbitrary\"],\"default\":[\"std\"],\"quickcheck\":[\"dep:quickcheck\"],\"rand\":[\"dep:rand\"],\"serde\":[\"dep:serde\"],\"std\":[\"num-integer/std\",\"num-traits/std\"]}}", + "num-conv_0.2.1": "{\"dependencies\":[],\"features\":{}}", "num-conv_0.2.2": "{\"dependencies\":[],\"features\":{}}", "num-integer_0.1.46": "{\"dependencies\":[{\"default_features\":false,\"features\":[\"i128\"],\"name\":\"num-traits\",\"req\":\"^0.2.11\"}],\"features\":{\"default\":[\"std\"],\"i128\":[],\"std\":[\"num-traits/std\"]}}", "num-iter_0.1.45": "{\"dependencies\":[{\"kind\":\"build\",\"name\":\"autocfg\",\"req\":\"^1\"},{\"default_features\":false,\"features\":[\"i128\"],\"name\":\"num-integer\",\"req\":\"^0.1.46\"},{\"default_features\":false,\"features\":[\"i128\"],\"name\":\"num-traits\",\"req\":\"^0.2.11\"}],\"features\":{\"default\":[\"std\"],\"i128\":[],\"std\":[\"num-integer/std\",\"num-traits/std\"]}}", "num-rational_0.4.2": "{\"dependencies\":[{\"default_features\":false,\"name\":\"num-bigint\",\"optional\":true,\"req\":\"^0.4.0\"},{\"default_features\":false,\"features\":[\"i128\"],\"name\":\"num-integer\",\"req\":\"^0.1.42\"},{\"default_features\":false,\"features\":[\"i128\"],\"name\":\"num-traits\",\"req\":\"^0.2.18\"},{\"default_features\":false,\"name\":\"serde\",\"optional\":true,\"req\":\"^1.0.0\"}],\"features\":{\"default\":[\"num-bigint\",\"std\"],\"num-bigint\":[\"dep:num-bigint\"],\"num-bigint-std\":[\"num-bigint/std\"],\"serde\":[\"dep:serde\"],\"std\":[\"num-bigint?/std\",\"num-integer/std\",\"num-traits/std\"]}}", "num-traits_0.2.19": "{\"dependencies\":[{\"kind\":\"build\",\"name\":\"autocfg\",\"req\":\"^1\"},{\"name\":\"libm\",\"optional\":true,\"req\":\"^0.2.0\"}],\"features\":{\"default\":[\"std\"],\"i128\":[],\"libm\":[\"dep:libm\"],\"std\":[]}}", + "num_cpus_1.17.0": "{\"dependencies\":[{\"name\":\"hermit-abi\",\"req\":\"^0.5.0\",\"target\":\"cfg(target_os = \\\"hermit\\\")\"},{\"name\":\"libc\",\"req\":\"^0.2.26\",\"target\":\"cfg(not(windows))\"}],\"features\":{}}", + "num_threads_0.1.7": "{\"dependencies\":[{\"name\":\"libc\",\"req\":\"^0.2.107\",\"target\":\"cfg(any(target_os = \\\"macos\\\", target_os = \\\"ios\\\", target_os = \\\"freebsd\\\"))\"}],\"features\":{}}", "once_cell_1.21.4": "{\"dependencies\":[{\"name\":\"critical-section\",\"optional\":true,\"req\":\"^1.1.3\"},{\"features\":[\"std\"],\"kind\":\"dev\",\"name\":\"critical-section\",\"req\":\"^1.1.3\"},{\"default_features\":false,\"name\":\"parking_lot_core\",\"optional\":true,\"req\":\"^0.9.10\"},{\"default_features\":false,\"name\":\"portable-atomic\",\"optional\":true,\"req\":\"^1.8\"},{\"kind\":\"dev\",\"name\":\"regex\",\"req\":\"^1.10.6\"}],\"features\":{\"alloc\":[\"race\"],\"atomic-polyfill\":[\"critical-section\"],\"critical-section\":[\"dep:critical-section\",\"portable-atomic\"],\"default\":[\"std\"],\"parking_lot\":[\"dep:parking_lot_core\"],\"portable-atomic\":[\"dep:portable-atomic\"],\"race\":[],\"std\":[\"alloc\"],\"unstable\":[]}}", "once_cell_polyfill_1.70.2": "{\"dependencies\":[],\"features\":{\"default\":[]}}", "openssl-probe_0.1.6": "{\"dependencies\":[],\"features\":{}}", @@ -1121,6 +1062,7 @@ "pkg-config_0.3.32": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"lazy_static\",\"req\":\"^1\"}],\"features\":{}}", "plain_0.2.3": "{\"dependencies\":[],\"features\":{}}", "portable-atomic-util_0.2.6": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"build-context\",\"req\":\"^0.1\"},{\"default_features\":false,\"features\":[\"require-cas\"],\"name\":\"portable-atomic\",\"req\":\"^1.5.1\"}],\"features\":{\"alloc\":[],\"default\":[],\"std\":[\"alloc\"]}}", + "portable-atomic-util_0.2.7": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"build-context\",\"req\":\"^0.1\"},{\"default_features\":false,\"features\":[\"require-cas\"],\"name\":\"portable-atomic\",\"req\":\"^1.5.1\"},{\"default_features\":false,\"name\":\"serde\",\"optional\":true,\"req\":\"^1.0.60\"}],\"features\":{\"alloc\":[],\"default\":[],\"std\":[\"alloc\"]}}", "portable-atomic_1.13.1": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"build-context\",\"req\":\"^0.1\"},{\"kind\":\"dev\",\"name\":\"crabgrind\",\"req\":\"^0.1\",\"target\":\"cfg(valgrind)\"},{\"name\":\"critical-section\",\"optional\":true,\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"crossbeam-utils\",\"req\":\"=0.8.16\"},{\"kind\":\"dev\",\"name\":\"fastrand\",\"req\":\"^2\"},{\"kind\":\"dev\",\"name\":\"libc\",\"req\":\"=0.2.163\",\"target\":\"cfg(unix)\"},{\"kind\":\"dev\",\"name\":\"paste\",\"req\":\"^1\"},{\"default_features\":false,\"kind\":\"dev\",\"name\":\"quickcheck\",\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"rustversion\",\"req\":\"^1\"},{\"default_features\":false,\"name\":\"serde\",\"optional\":true,\"req\":\"^1.0.60\"},{\"kind\":\"dev\",\"name\":\"sptr\",\"req\":\"^0.3\"},{\"kind\":\"dev\",\"name\":\"static_assertions\",\"req\":\"^1\"},{\"features\":[\"Win32_Foundation\",\"Win32_System_Threading\"],\"kind\":\"dev\",\"name\":\"windows-sys\",\"req\":\"^0.61\",\"target\":\"cfg(windows)\"}],\"features\":{\"default\":[\"fallback\"],\"disable-fiq\":[],\"fallback\":[],\"float\":[],\"force-amo\":[],\"require-cas\":[],\"s-mode\":[],\"std\":[],\"unsafe-assume-privileged\":[],\"unsafe-assume-single-core\":[]}}", "potential_utf_0.1.5": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"bincode\",\"req\":\"^1.3.1\"},{\"default_features\":false,\"name\":\"databake\",\"optional\":true,\"req\":\"^0.2.0\"},{\"default_features\":false,\"name\":\"serde_core\",\"optional\":true,\"req\":\"^1.0.220\"},{\"kind\":\"dev\",\"name\":\"serde_json\",\"req\":\"^1.0.45\"},{\"default_features\":false,\"name\":\"writeable\",\"optional\":true,\"req\":\"^0.6.1\"},{\"default_features\":false,\"name\":\"zerovec\",\"optional\":true,\"req\":\"^0.11.6\"}],\"features\":{\"alloc\":[\"serde_core?/alloc\",\"writeable/alloc\",\"zerovec?/alloc\"],\"databake\":[\"dep:databake\"],\"default\":[\"alloc\"],\"serde\":[\"dep:serde_core\"],\"writeable\":[\"dep:writeable\"],\"zerovec\":[\"dep:zerovec\"]}}", "powerfmt_0.2.0": "{\"dependencies\":[{\"name\":\"powerfmt-macros\",\"optional\":true,\"req\":\"=0.1.0\"}],\"features\":{\"alloc\":[],\"default\":[\"std\",\"macros\"],\"macros\":[\"dep:powerfmt-macros\"],\"std\":[\"alloc\"]}}", @@ -1146,7 +1088,8 @@ "pyo3-macros-backend_0.28.3": "{\"dependencies\":[{\"name\":\"heck\",\"req\":\"^0.5\"},{\"default_features\":false,\"name\":\"proc-macro2\",\"req\":\"^1.0.60\"},{\"features\":[\"resolve-config\"],\"name\":\"pyo3-build-config\",\"req\":\"=0.28.3\"},{\"kind\":\"build\",\"name\":\"pyo3-build-config\",\"req\":\"=0.28.3\"},{\"default_features\":false,\"name\":\"quote\",\"req\":\"^1.0.37\"},{\"default_features\":false,\"features\":[\"derive\",\"parsing\",\"printing\",\"clone-impls\",\"full\",\"extra-traits\",\"visit-mut\"],\"name\":\"syn\",\"req\":\"^2.0.59\"}],\"features\":{\"experimental-async\":[],\"experimental-inspect\":[]}}", "pyo3-macros_0.28.3": "{\"dependencies\":[{\"default_features\":false,\"name\":\"proc-macro2\",\"req\":\"^1.0.60\"},{\"name\":\"pyo3-macros-backend\",\"req\":\"=0.28.3\"},{\"name\":\"quote\",\"req\":\"^1\"},{\"features\":[\"full\",\"extra-traits\"],\"name\":\"syn\",\"req\":\"^2\"}],\"features\":{\"experimental-async\":[\"pyo3-macros-backend/experimental-async\"],\"experimental-inspect\":[\"pyo3-macros-backend/experimental-inspect\"],\"multiple-pymethods\":[]}}", "pyo3_0.28.3": "{\"dependencies\":[{\"name\":\"anyhow\",\"optional\":true,\"req\":\"^1.0.1\"},{\"kind\":\"dev\",\"name\":\"assert_approx_eq\",\"req\":\"^1.1.0\"},{\"name\":\"bigdecimal\",\"optional\":true,\"req\":\"^0.4.7\"},{\"name\":\"bytes\",\"optional\":true,\"req\":\"^1.10\"},{\"default_features\":false,\"name\":\"chrono\",\"optional\":true,\"req\":\"^0.4.25\"},{\"kind\":\"dev\",\"name\":\"chrono\",\"req\":\"^0.4.25\"},{\"default_features\":false,\"name\":\"chrono-tz\",\"optional\":true,\"req\":\">=0.10, <0.11\"},{\"kind\":\"dev\",\"name\":\"chrono-tz\",\"req\":\">=0.10, <0.11\"},{\"name\":\"either\",\"optional\":true,\"req\":\"^1.9\"},{\"name\":\"eyre\",\"optional\":true,\"req\":\">=0.6.8, <0.7\"},{\"kind\":\"dev\",\"name\":\"futures\",\"req\":\"^0.3.28\"},{\"default_features\":false,\"name\":\"hashbrown\",\"optional\":true,\"req\":\">=0.15.0, <0.17\"},{\"features\":[\"fallback\"],\"name\":\"iana-time-zone\",\"optional\":true,\"req\":\"^0.1\"},{\"name\":\"indexmap\",\"optional\":true,\"req\":\">=2.5.0, <3\"},{\"name\":\"inventory\",\"optional\":true,\"req\":\"^0.3.5\"},{\"name\":\"jiff-02\",\"optional\":true,\"package\":\"jiff\",\"req\":\"^0.2\"},{\"name\":\"libc\",\"req\":\"^0.2.62\"},{\"name\":\"lock_api\",\"optional\":true,\"req\":\"^0.4\"},{\"name\":\"num-bigint\",\"optional\":true,\"req\":\"^0.4.4\"},{\"name\":\"num-complex\",\"optional\":true,\"req\":\">=0.4.6, <0.5\"},{\"name\":\"num-rational\",\"optional\":true,\"req\":\"^0.4.1\"},{\"name\":\"num-traits\",\"optional\":true,\"req\":\"^0.2.16\"},{\"name\":\"once_cell\",\"req\":\"^1.21\"},{\"default_features\":false,\"name\":\"ordered-float\",\"optional\":true,\"req\":\"^5.0.0\"},{\"name\":\"parking_lot\",\"optional\":true,\"req\":\"^0.12\"},{\"features\":[\"arc_lock\"],\"kind\":\"dev\",\"name\":\"parking_lot\",\"req\":\"^0.12.3\"},{\"name\":\"portable-atomic\",\"req\":\"^1.0\",\"target\":\"cfg(not(target_has_atomic = \\\"64\\\"))\"},{\"default_features\":false,\"features\":[\"std\"],\"kind\":\"dev\",\"name\":\"proptest\",\"req\":\"^1.0\"},{\"features\":[\"resolve-config\"],\"kind\":\"build\",\"name\":\"pyo3-build-config\",\"req\":\"=0.28.3\"},{\"name\":\"pyo3-ffi\",\"req\":\"=0.28.3\"},{\"name\":\"pyo3-macros\",\"optional\":true,\"req\":\"=0.28.3\"},{\"kind\":\"dev\",\"name\":\"rayon\",\"req\":\"^1.6.1\"},{\"default_features\":false,\"name\":\"rust_decimal\",\"optional\":true,\"req\":\"^1.15\"},{\"kind\":\"dev\",\"name\":\"send_wrapper\",\"req\":\"^0.6\"},{\"name\":\"serde\",\"optional\":true,\"req\":\"^1.0\"},{\"features\":[\"derive\"],\"kind\":\"dev\",\"name\":\"serde\",\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"serde_json\",\"req\":\"^1.0.61\"},{\"name\":\"smallvec\",\"optional\":true,\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"static_assertions\",\"req\":\"^1.1.0\"},{\"kind\":\"dev\",\"name\":\"tempfile\",\"req\":\"^3.12.0\"},{\"default_features\":false,\"name\":\"time\",\"optional\":true,\"req\":\"^0.3.38\"},{\"kind\":\"dev\",\"name\":\"trybuild\",\"req\":\">=1.0.115\"},{\"name\":\"uuid\",\"optional\":true,\"req\":\"^1.12.0\"},{\"features\":[\"v4\"],\"kind\":\"dev\",\"name\":\"uuid\",\"req\":\"^1.10.0\"}],\"features\":{\"abi3\":[\"pyo3-build-config/abi3\",\"pyo3-ffi/abi3\"],\"abi3-py310\":[\"abi3-py311\",\"pyo3-build-config/abi3-py310\",\"pyo3-ffi/abi3-py310\"],\"abi3-py311\":[\"abi3-py312\",\"pyo3-build-config/abi3-py311\",\"pyo3-ffi/abi3-py311\"],\"abi3-py312\":[\"abi3-py313\",\"pyo3-build-config/abi3-py312\",\"pyo3-ffi/abi3-py312\"],\"abi3-py313\":[\"abi3-py314\",\"pyo3-build-config/abi3-py313\",\"pyo3-ffi/abi3-py313\"],\"abi3-py314\":[\"abi3\",\"pyo3-build-config/abi3-py314\",\"pyo3-ffi/abi3-py314\"],\"abi3-py37\":[\"abi3-py38\",\"pyo3-build-config/abi3-py37\",\"pyo3-ffi/abi3-py37\"],\"abi3-py38\":[\"abi3-py39\",\"pyo3-build-config/abi3-py38\",\"pyo3-ffi/abi3-py38\"],\"abi3-py39\":[\"abi3-py310\",\"pyo3-build-config/abi3-py39\",\"pyo3-ffi/abi3-py39\"],\"arc_lock\":[\"lock_api\",\"lock_api/arc_lock\",\"parking_lot?/arc_lock\"],\"auto-initialize\":[],\"bigdecimal\":[\"dep:bigdecimal\",\"num-bigint\"],\"chrono-local\":[\"chrono/clock\",\"dep:iana-time-zone\"],\"default\":[\"macros\"],\"experimental-async\":[\"macros\",\"pyo3-macros/experimental-async\"],\"experimental-inspect\":[\"pyo3-macros/experimental-inspect\"],\"extension-module\":[\"pyo3-ffi/extension-module\"],\"full\":[\"macros\",\"anyhow\",\"arc_lock\",\"bigdecimal\",\"bytes\",\"chrono\",\"chrono-local\",\"chrono-tz\",\"either\",\"experimental-async\",\"experimental-inspect\",\"eyre\",\"hashbrown\",\"indexmap\",\"jiff-02\",\"lock_api\",\"num-bigint\",\"num-complex\",\"num-rational\",\"ordered-float\",\"parking_lot\",\"py-clone\",\"rust_decimal\",\"serde\",\"smallvec\",\"time\",\"uuid\"],\"generate-import-lib\":[\"pyo3-ffi/generate-import-lib\"],\"macros\":[\"pyo3-macros\"],\"multiple-pymethods\":[\"inventory\",\"pyo3-macros/multiple-pymethods\"],\"nightly\":[],\"num-bigint\":[\"dep:num-bigint\",\"dep:num-traits\"],\"parking_lot\":[\"dep:parking_lot\",\"lock_api\"],\"py-clone\":[]}}", - "quick-xml_0.39.4": "{\"dependencies\":[{\"features\":[\"derive\"],\"name\":\"arbitrary\",\"optional\":true,\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"criterion\",\"req\":\">=0.4, <0.9\"},{\"name\":\"document-features\",\"optional\":true,\"req\":\"^0.2\"},{\"name\":\"encoding_rs\",\"optional\":true,\"req\":\"^0.8\"},{\"name\":\"memchr\",\"req\":\"^2.1\"},{\"kind\":\"dev\",\"name\":\"pretty_assertions\",\"req\":\"^1.4\"},{\"kind\":\"dev\",\"name\":\"regex\",\"req\":\"^1\"},{\"name\":\"serde\",\"optional\":true,\"req\":\">=1.0.180\"},{\"kind\":\"dev\",\"name\":\"serde-value\",\"req\":\"^0.7\"},{\"kind\":\"dev\",\"name\":\"serde_derive\",\"req\":\"^1.0.206\"},{\"default_features\":false,\"features\":[\"io-util\"],\"name\":\"tokio\",\"optional\":true,\"req\":\"^1.10\"},{\"default_features\":false,\"features\":[\"macros\",\"rt\"],\"kind\":\"dev\",\"name\":\"tokio\",\"req\":\"^1.21\"},{\"kind\":\"dev\",\"name\":\"tokio-test\",\"req\":\"^0.4\"}],\"features\":{\"async-tokio\":[\"tokio\"],\"default\":[],\"encoding\":[\"encoding_rs\"],\"escape-html\":[],\"overlapped-lists\":[],\"serde-types\":[\"serde/derive\"],\"serialize\":[\"serde\"]}}", + "quick-error_1.2.3": "{\"dependencies\":[],\"features\":{}}", + "quick-xml_0.41.0": "{\"dependencies\":[{\"features\":[\"derive\"],\"name\":\"arbitrary\",\"optional\":true,\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"criterion\",\"req\":\">=0.4, <0.9\"},{\"name\":\"document-features\",\"optional\":true,\"req\":\"^0.2\"},{\"name\":\"encoding_rs\",\"optional\":true,\"req\":\"^0.8\"},{\"name\":\"memchr\",\"req\":\"^2.1\"},{\"kind\":\"dev\",\"name\":\"pretty_assertions\",\"req\":\"^1.4\"},{\"kind\":\"dev\",\"name\":\"regex\",\"req\":\"^1\"},{\"name\":\"serde\",\"optional\":true,\"req\":\">=1.0.180\"},{\"kind\":\"dev\",\"name\":\"serde-value\",\"req\":\"^0.7\"},{\"kind\":\"dev\",\"name\":\"serde_derive\",\"req\":\"^1.0.206\"},{\"default_features\":false,\"features\":[\"io-util\"],\"name\":\"tokio\",\"optional\":true,\"req\":\"^1.10\"},{\"default_features\":false,\"features\":[\"macros\",\"rt\"],\"kind\":\"dev\",\"name\":\"tokio\",\"req\":\"^1.21\"},{\"kind\":\"dev\",\"name\":\"tokio-test\",\"req\":\"^0.4\"}],\"features\":{\"async-tokio\":[\"tokio\"],\"default\":[],\"encoding\":[\"encoding_rs\"],\"escape-html\":[],\"overlapped-lists\":[],\"serde-types\":[\"serde/derive\"],\"serialize\":[\"serde\"]}}", "quote_1.0.45": "{\"dependencies\":[{\"default_features\":false,\"name\":\"proc-macro2\",\"req\":\"^1.0.80\"},{\"kind\":\"dev\",\"name\":\"rustversion\",\"req\":\"^1.0\"},{\"features\":[\"diff\"],\"kind\":\"dev\",\"name\":\"trybuild\",\"req\":\"^1.0.108\"}],\"features\":{\"default\":[\"proc-macro\"],\"proc-macro\":[\"proc-macro2/proc-macro\"]}}", "r-efi_5.3.0": "{\"dependencies\":[{\"name\":\"core\",\"optional\":true,\"package\":\"rustc-std-workspace-core\",\"req\":\"^1.0.0\"}],\"features\":{\"efiapi\":[],\"examples\":[\"native\"],\"native\":[],\"rustc-dep-of-std\":[\"core\"]}}", "r-efi_6.0.0": "{\"dependencies\":[{\"name\":\"core\",\"optional\":true,\"package\":\"rustc-std-workspace-core\",\"req\":\"^1.0.0\"}],\"features\":{\"native\":[],\"rustc-dep-of-std\":[\"core\"]}}", @@ -1159,6 +1102,8 @@ "rand_core_0.10.1": "{\"dependencies\":[],\"features\":{}}", "rand_core_0.6.4": "{\"dependencies\":[{\"name\":\"getrandom\",\"optional\":true,\"req\":\"^0.2\"},{\"features\":[\"derive\"],\"name\":\"serde\",\"optional\":true,\"req\":\"^1\"}],\"features\":{\"alloc\":[],\"serde1\":[\"serde\"],\"std\":[\"alloc\",\"getrandom\",\"getrandom/std\"]}}", "rand_core_0.9.3": "{\"dependencies\":[{\"name\":\"getrandom\",\"optional\":true,\"req\":\"^0.3.0\"},{\"features\":[\"derive\"],\"name\":\"serde\",\"optional\":true,\"req\":\"^1\"}],\"features\":{\"os_rng\":[\"dep:getrandom\"],\"serde\":[\"dep:serde\"],\"std\":[\"getrandom?/std\"]}}", + "rayon-core_1.13.0": "{\"dependencies\":[{\"name\":\"crossbeam-deque\",\"req\":\"^0.8.1\"},{\"name\":\"crossbeam-utils\",\"req\":\"^0.8.0\"},{\"kind\":\"dev\",\"name\":\"libc\",\"req\":\"^0.2\",\"target\":\"cfg(unix)\"},{\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.9\"},{\"kind\":\"dev\",\"name\":\"rand_xorshift\",\"req\":\"^0.4\"},{\"kind\":\"dev\",\"name\":\"scoped-tls\",\"req\":\"^1.0\"},{\"name\":\"wasm_sync\",\"optional\":true,\"req\":\"^0.1.0\"}],\"features\":{\"web_spin_lock\":[\"dep:wasm_sync\"]}}", + "rayon_1.12.0": "{\"dependencies\":[{\"default_features\":false,\"name\":\"either\",\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.9\"},{\"kind\":\"dev\",\"name\":\"rand_xorshift\",\"req\":\"^0.4\"},{\"name\":\"rayon-core\",\"req\":\"^1.13.0\"},{\"name\":\"wasm_sync\",\"optional\":true,\"req\":\"^0.1.0\"}],\"features\":{\"web_spin_lock\":[\"dep:wasm_sync\",\"rayon-core/web_spin_lock\"]}}", "redis-protocol_6.0.0": "{\"dependencies\":[{\"default_features\":false,\"name\":\"bytes\",\"optional\":true,\"req\":\"^1.1\"},{\"default_features\":false,\"name\":\"bytes-utils\",\"optional\":true,\"req\":\"^0.1\"},{\"default_features\":false,\"name\":\"cookie-factory\",\"req\":\"=0.3.2\"},{\"name\":\"crc16\",\"req\":\"^0.4\"},{\"features\":[\"html_reports\"],\"kind\":\"dev\",\"name\":\"criterion\",\"req\":\"^0.4\"},{\"kind\":\"dev\",\"name\":\"futures\",\"req\":\"^0.3\"},{\"name\":\"hashbrown\",\"optional\":true,\"req\":\"^0.14\"},{\"name\":\"indexmap\",\"optional\":true,\"req\":\"^2.2\"},{\"kind\":\"dev\",\"name\":\"itertools\",\"req\":\"^0.12\"},{\"name\":\"libm\",\"optional\":true,\"req\":\"^0.2\"},{\"name\":\"log\",\"req\":\"^0.4\"},{\"default_features\":false,\"name\":\"nom\",\"req\":\"^7.1\"},{\"kind\":\"dev\",\"name\":\"pretty_env_logger\",\"req\":\"^0.5\"},{\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.8\"},{\"features\":[\"full\"],\"kind\":\"dev\",\"name\":\"tokio\",\"req\":\"^1.36\"},{\"features\":[\"codec\"],\"name\":\"tokio-util\",\"optional\":true,\"req\":\"^0.7\"}],\"features\":{\"alloc\":[\"nom/alloc\"],\"bytes\":[\"dep:bytes\",\"bytes-utils\"],\"codec\":[\"tokio-util\",\"bytes\"],\"convert\":[],\"decode-logs\":[],\"default\":[\"std\",\"resp2\",\"resp3\"],\"index-map\":[\"indexmap\"],\"resp2\":[],\"resp3\":[],\"std\":[\"cookie-factory/default\",\"nom/default\"]}}", "redis-test_1.0.0": "{\"dependencies\":[{\"name\":\"bytes\",\"optional\":true,\"req\":\"^1\"},{\"name\":\"futures\",\"optional\":true,\"req\":\"^0.3\"},{\"name\":\"rand\",\"req\":\"^0.9\"},{\"default_features\":false,\"name\":\"redis\",\"req\":\"^1\"},{\"features\":[\"aio\",\"tokio-comp\"],\"kind\":\"dev\",\"name\":\"redis\",\"req\":\"^1\"},{\"name\":\"socket2\",\"req\":\"^0.6\"},{\"name\":\"tempfile\",\"req\":\"^3.23.0\"},{\"features\":[\"rt\",\"macros\",\"rt-multi-thread\",\"test-util\",\"time\"],\"kind\":\"dev\",\"name\":\"tokio\",\"req\":\"^1\"}],\"features\":{\"aio\":[\"futures\",\"redis/aio\"]}}", "redis_1.0.0": "{\"dependencies\":[{\"name\":\"ahash\",\"optional\":true,\"req\":\"^0.8.11\"},{\"kind\":\"dev\",\"name\":\"anyhow\",\"req\":\"^1\"},{\"name\":\"arc-swap\",\"optional\":true,\"req\":\"^1.7.1\"},{\"name\":\"arcstr\",\"req\":\"^1.2.0\"},{\"kind\":\"dev\",\"name\":\"assert_approx_eq\",\"req\":\"^1.0\"},{\"name\":\"async-io\",\"optional\":true,\"req\":\"^2\"},{\"name\":\"async-native-tls\",\"optional\":true,\"req\":\"^0.5\"},{\"default_features\":false,\"name\":\"backon\",\"optional\":true,\"req\":\"^1.6.0\"},{\"name\":\"bb8\",\"optional\":true,\"req\":\"^0.9.1\"},{\"name\":\"bigdecimal\",\"optional\":true,\"req\":\"^0.4.9\"},{\"name\":\"bytes\",\"optional\":true,\"req\":\"^1\"},{\"name\":\"cfg-if\",\"optional\":true,\"req\":\"^1\"},{\"default_features\":false,\"features\":[\"std\"],\"name\":\"combine\",\"req\":\"^4.6\"},{\"name\":\"crc16\",\"optional\":true,\"req\":\"^0.4\"},{\"kind\":\"dev\",\"name\":\"criterion\",\"req\":\"^0.7\"},{\"kind\":\"dev\",\"name\":\"fnv\",\"req\":\"^1.0.5\"},{\"kind\":\"dev\",\"name\":\"futures\",\"req\":\"^0.3\"},{\"name\":\"futures-channel\",\"optional\":true,\"req\":\"^0.3.31\"},{\"default_features\":false,\"name\":\"futures-rustls\",\"optional\":true,\"req\":\"^0.26\"},{\"kind\":\"dev\",\"name\":\"futures-time\",\"req\":\"^3\"},{\"default_features\":false,\"features\":[\"std\",\"sink\"],\"name\":\"futures-util\",\"optional\":true,\"req\":\"^0.3.31\"},{\"name\":\"hashbrown\",\"optional\":true,\"req\":\"^0.16\"},{\"name\":\"itoa\",\"req\":\"^1.0\"},{\"name\":\"log\",\"optional\":true,\"req\":\"^0.4\"},{\"name\":\"lru\",\"optional\":true,\"req\":\"^0.16\"},{\"name\":\"native-tls\",\"optional\":true,\"req\":\"^0.2\"},{\"name\":\"num-bigint\",\"optional\":true,\"req\":\"^0.4.6\"},{\"features\":[\"tokio\",\"quickcheck1\"],\"kind\":\"dev\",\"name\":\"partial-io\",\"req\":\"^0.5\"},{\"name\":\"percent-encoding\",\"req\":\"^2.3\"},{\"name\":\"pin-project-lite\",\"optional\":true,\"req\":\"^0.2\"},{\"kind\":\"dev\",\"name\":\"quickcheck\",\"req\":\"^1.0.3\"},{\"name\":\"r2d2\",\"optional\":true,\"req\":\"^0.8.10\"},{\"name\":\"rand\",\"optional\":true,\"req\":\"^0.9\"},{\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.9\"},{\"kind\":\"dev\",\"name\":\"rstest\",\"req\":\"^0.26\"},{\"name\":\"rust_decimal\",\"optional\":true,\"req\":\"^1.39.0\"},{\"default_features\":false,\"name\":\"rustls\",\"optional\":true,\"req\":\"^0.23\"},{\"features\":[\"ring\"],\"kind\":\"dev\",\"name\":\"rustls\",\"req\":\"^0.23\"},{\"name\":\"rustls-native-certs\",\"optional\":true,\"req\":\"^0.8\"},{\"name\":\"ryu\",\"req\":\"^1.0\"},{\"name\":\"serde\",\"optional\":true,\"req\":\"^1.0.219\"},{\"name\":\"serde_json\",\"optional\":true,\"req\":\"^1.0.145\"},{\"name\":\"sha1_smol\",\"optional\":true,\"req\":\"^1.0\"},{\"name\":\"smol\",\"optional\":true,\"req\":\"^2\"},{\"name\":\"smol-timeout\",\"optional\":true,\"req\":\"^0.6\"},{\"features\":[\"all\"],\"name\":\"socket2\",\"req\":\"^0.6\",\"target\":\"cfg(not(target_family = \\\"wasm\\\"))\"},{\"kind\":\"dev\",\"name\":\"tempfile\",\"req\":\"^3.23.0\"},{\"features\":[\"rt\",\"net\",\"time\",\"sync\"],\"name\":\"tokio\",\"optional\":true,\"req\":\"^1\"},{\"features\":[\"rt\",\"macros\",\"rt-multi-thread\",\"test-util\",\"time\"],\"kind\":\"dev\",\"name\":\"tokio\",\"req\":\"^1\"},{\"name\":\"tokio-native-tls\",\"optional\":true,\"req\":\"^0.3\"},{\"default_features\":false,\"name\":\"tokio-rustls\",\"optional\":true,\"req\":\"^0.26\"},{\"name\":\"tokio-util\",\"optional\":true,\"req\":\"^0.7\"},{\"name\":\"url\",\"req\":\"^2.5\"},{\"name\":\"uuid\",\"optional\":true,\"req\":\"^1.18.1\"},{\"name\":\"webpki-roots\",\"optional\":true,\"req\":\"^1.0\"},{\"features\":[\"xxh3\"],\"name\":\"xxhash-rust\",\"req\":\"^0.8\"}],\"features\":{\"acl\":[],\"aio\":[\"bytes\",\"dep:pin-project-lite\",\"dep:futures-util\",\"dep:tokio\",\"dep:tokio-util\",\"tokio-util/codec\",\"combine/tokio\",\"dep:cfg-if\"],\"bb8\":[\"dep:bb8\"],\"cache-aio\":[\"aio\",\"dep:lru\"],\"cluster\":[\"dep:crc16\",\"dep:rand\"],\"cluster-async\":[\"aio\",\"cluster\",\"dep:log\"],\"connection-manager\":[\"dep:arc-swap\",\"dep:futures-channel\",\"aio\",\"dep:backon\"],\"default\":[\"acl\",\"streams\",\"geospatial\",\"script\",\"num-bigint\"],\"geospatial\":[],\"json\":[\"dep:serde\",\"serde/derive\",\"dep:serde_json\"],\"num-bigint\":[\"dep:num-bigint\"],\"r2d2\":[\"dep:r2d2\"],\"script\":[\"dep:sha1_smol\"],\"sentinel\":[\"dep:rand\"],\"smol-comp\":[\"aio\",\"dep:smol\",\"dep:smol-timeout\",\"dep:async-io\"],\"smol-native-tls-comp\":[\"smol-comp\",\"dep:async-native-tls\",\"tls-native-tls\"],\"smol-rustls-comp\":[\"smol-comp\",\"dep:futures-rustls\",\"tls-rustls\"],\"streams\":[],\"tls-native-tls\":[\"dep:native-tls\"],\"tls-rustls\":[\"dep:rustls\",\"rustls/std\",\"dep:rustls-native-certs\"],\"tls-rustls-insecure\":[\"tls-rustls\"],\"tls-rustls-webpki-roots\":[\"tls-rustls\",\"dep:webpki-roots\"],\"tokio-comp\":[\"aio\",\"tokio/net\"],\"tokio-native-tls-comp\":[\"tokio-comp\",\"tls-native-tls\",\"dep:tokio-native-tls\"],\"tokio-rustls-comp\":[\"tokio-comp\",\"tls-rustls\",\"dep:tokio-rustls\"],\"vector-sets\":[\"dep:serde\",\"serde/derive\",\"dep:serde_json\"]}}", @@ -1178,21 +1123,26 @@ "rfc6979_0.4.0": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"hex-literal\",\"req\":\"^0.3\"},{\"default_features\":false,\"features\":[\"reset\"],\"name\":\"hmac\",\"req\":\"^0.12\"},{\"kind\":\"dev\",\"name\":\"sha2\",\"req\":\"^0.10\"},{\"default_features\":false,\"name\":\"subtle\",\"req\":\"^2\"}],\"features\":{}}", "ring_0.17.14": "{\"dependencies\":[{\"default_features\":false,\"kind\":\"build\",\"name\":\"cc\",\"req\":\"^1.2.8\"},{\"default_features\":false,\"name\":\"cfg-if\",\"req\":\"^1.0.0\"},{\"name\":\"getrandom\",\"req\":\"^0.2.10\"},{\"default_features\":false,\"name\":\"libc\",\"req\":\"^0.2.148\",\"target\":\"cfg(all(any(all(target_arch = \\\"aarch64\\\", target_endian = \\\"little\\\"), all(target_arch = \\\"arm\\\", target_endian = \\\"little\\\")), any(target_os = \\\"android\\\", target_os = \\\"linux\\\")))\"},{\"default_features\":false,\"name\":\"libc\",\"req\":\"^0.2.155\",\"target\":\"cfg(all(all(target_arch = \\\"aarch64\\\", target_endian = \\\"little\\\"), target_vendor = \\\"apple\\\", any(target_os = \\\"ios\\\", target_os = \\\"macos\\\", target_os = \\\"tvos\\\", target_os = \\\"visionos\\\", target_os = \\\"watchos\\\")))\"},{\"default_features\":false,\"kind\":\"dev\",\"name\":\"libc\",\"req\":\"^0.2.148\",\"target\":\"cfg(any(unix, windows, target_os = \\\"wasi\\\"))\"},{\"name\":\"untrusted\",\"req\":\"^0.9\"},{\"default_features\":false,\"features\":[\"std\"],\"kind\":\"dev\",\"name\":\"wasm-bindgen-test\",\"req\":\"^0.3.37\",\"target\":\"cfg(all(target_arch = \\\"wasm32\\\", target_os = \\\"unknown\\\"))\"},{\"features\":[\"Win32_Foundation\",\"Win32_System_Threading\"],\"name\":\"windows-sys\",\"req\":\"^0.52\",\"target\":\"cfg(all(all(target_arch = \\\"aarch64\\\", target_endian = \\\"little\\\"), target_os = \\\"windows\\\"))\"}],\"features\":{\"alloc\":[],\"default\":[\"alloc\",\"dev_urandom_fallback\"],\"dev_urandom_fallback\":[],\"less-safe-getrandom-custom-or-rdrand\":[],\"less-safe-getrandom-espidf\":[],\"slow_tests\":[],\"std\":[\"alloc\"],\"test_logging\":[],\"unstable-testing-arm-no-hw\":[],\"unstable-testing-arm-no-neon\":[],\"wasm32_unknown_unknown_js\":[\"getrandom/js\"]}}", "rlimit_0.10.2": "{\"dependencies\":[{\"name\":\"libc\",\"req\":\"^0.2.147\"},{\"kind\":\"dev\",\"name\":\"once_cell\",\"req\":\"^1.18.0\"}],\"features\":{}}", + "rouille_3.6.2": "{\"dependencies\":[{\"name\":\"base64\",\"req\":\"^0.13\"},{\"name\":\"brotli\",\"optional\":true,\"req\":\"^3.3.2\"},{\"default_features\":false,\"features\":[\"clock\"],\"name\":\"chrono\",\"req\":\"^0.4.19\"},{\"features\":[\"gzip\"],\"name\":\"deflate\",\"optional\":true,\"req\":\"^1.0.0\"},{\"name\":\"filetime\",\"req\":\"^0.2.0\"},{\"kind\":\"dev\",\"name\":\"log\",\"req\":\"^0.4\"},{\"default_features\":false,\"features\":[\"server\"],\"name\":\"multipart\",\"req\":\"^0.18\"},{\"name\":\"percent-encoding\",\"req\":\"^2\"},{\"default_features\":false,\"kind\":\"dev\",\"name\":\"postgres\",\"req\":\"^0.19\"},{\"name\":\"rand\",\"req\":\"^0.8\"},{\"name\":\"serde\",\"req\":\"^1\"},{\"name\":\"serde_derive\",\"req\":\"^1\"},{\"name\":\"serde_json\",\"req\":\"^1\"},{\"name\":\"sha1_smol\",\"req\":\"^1.0.0\"},{\"name\":\"threadpool\",\"req\":\"^1\"},{\"features\":[\"local-offset\"],\"name\":\"time\",\"req\":\"^0.3.15\"},{\"default_features\":false,\"name\":\"tiny_http\",\"req\":\"^0.12.0\"},{\"name\":\"url\",\"req\":\"^2\"}],\"features\":{\"default\":[\"gzip\",\"brotli\"],\"gzip\":[\"deflate\"],\"rustls\":[\"tiny_http/ssl-rustls\"],\"ssl\":[\"tiny_http/ssl\"]}}", "roxmltree_0.14.1": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"pretty_assertions\",\"req\":\"^0.5\"},{\"name\":\"xmlparser\",\"req\":\"^0.13.3\"}],\"features\":{\"default\":[\"std\"],\"std\":[]}}", "rsa_0.9.10": "{\"dependencies\":[{\"features\":[\"alloc\"],\"kind\":\"dev\",\"name\":\"base64ct\",\"req\":\"^1\"},{\"default_features\":false,\"name\":\"const-oid\",\"req\":\"^0.9\"},{\"default_features\":false,\"features\":[\"alloc\",\"oid\"],\"name\":\"digest\",\"req\":\"^0.10.5\"},{\"kind\":\"dev\",\"name\":\"hex-literal\",\"req\":\"^0.4.1\"},{\"default_features\":false,\"features\":[\"i128\",\"prime\",\"zeroize\"],\"name\":\"num-bigint\",\"package\":\"num-bigint-dig\",\"req\":\"^0.8.6\"},{\"default_features\":false,\"name\":\"num-integer\",\"req\":\"^0.1.39\"},{\"default_features\":false,\"features\":[\"libm\"],\"name\":\"num-traits\",\"req\":\"^0.2.9\"},{\"default_features\":false,\"features\":[\"alloc\",\"pkcs8\"],\"name\":\"pkcs1\",\"req\":\"^0.7.5\"},{\"default_features\":false,\"features\":[\"alloc\"],\"name\":\"pkcs8\",\"req\":\"^0.10.2\"},{\"kind\":\"dev\",\"name\":\"proptest\",\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.8\"},{\"kind\":\"dev\",\"name\":\"rand_chacha\",\"req\":\"^0.3\"},{\"default_features\":false,\"name\":\"rand_core\",\"req\":\"^0.6.4\"},{\"default_features\":false,\"kind\":\"dev\",\"name\":\"rand_core\",\"req\":\"^0.6\"},{\"kind\":\"dev\",\"name\":\"rand_xorshift\",\"req\":\"^0.3\"},{\"default_features\":false,\"features\":[\"derive\"],\"name\":\"serde\",\"optional\":true,\"req\":\"^1.0.184\"},{\"kind\":\"dev\",\"name\":\"serde_test\",\"req\":\"^1.0.89\"},{\"default_features\":false,\"features\":[\"oid\"],\"name\":\"sha1\",\"optional\":true,\"req\":\"^0.10.5\"},{\"default_features\":false,\"features\":[\"oid\"],\"kind\":\"dev\",\"name\":\"sha1\",\"req\":\"^0.10.5\"},{\"default_features\":false,\"features\":[\"oid\"],\"name\":\"sha2\",\"optional\":true,\"req\":\"^0.10.6\"},{\"default_features\":false,\"features\":[\"oid\"],\"kind\":\"dev\",\"name\":\"sha2\",\"req\":\"^0.10.6\"},{\"default_features\":false,\"features\":[\"oid\"],\"kind\":\"dev\",\"name\":\"sha3\",\"req\":\"^0.10.7\"},{\"default_features\":false,\"features\":[\"alloc\",\"digest\",\"rand_core\"],\"name\":\"signature\",\"req\":\">2.0, <2.3\"},{\"default_features\":false,\"features\":[\"alloc\"],\"name\":\"spki\",\"req\":\"^0.7.3\"},{\"default_features\":false,\"name\":\"subtle\",\"req\":\"^2.1.1\"},{\"features\":[\"alloc\"],\"name\":\"zeroize\",\"req\":\"^1.5\"}],\"features\":{\"default\":[\"std\",\"pem\",\"u64_digit\"],\"getrandom\":[\"rand_core/getrandom\"],\"hazmat\":[],\"nightly\":[\"num-bigint/nightly\"],\"pem\":[\"pkcs1/pem\",\"pkcs8/pem\"],\"pkcs5\":[\"pkcs8/encryption\"],\"serde\":[\"dep:serde\",\"num-bigint/serde\"],\"std\":[\"digest/std\",\"pkcs1/std\",\"pkcs8/std\",\"rand_core/std\",\"signature/std\"],\"u64_digit\":[\"num-bigint/u64_digit\"]}}", "rust_decimal_1.39.0": "{\"dependencies\":[{\"default_features\":false,\"name\":\"arbitrary\",\"optional\":true,\"req\":\"^1.0\"},{\"default_features\":false,\"name\":\"arrayvec\",\"req\":\"^0.7\"},{\"default_features\":false,\"kind\":\"dev\",\"name\":\"bincode\",\"req\":\"^1.0\"},{\"default_features\":false,\"features\":[\"derive\",\"unstable__schema\"],\"name\":\"borsh\",\"optional\":true,\"req\":\"^1.1.1\"},{\"default_features\":false,\"name\":\"bytes\",\"optional\":true,\"req\":\"^1.0\"},{\"default_features\":false,\"kind\":\"dev\",\"name\":\"bytes\",\"req\":\"^1.0\"},{\"default_features\":false,\"kind\":\"dev\",\"name\":\"criterion\",\"req\":\"^0.5\"},{\"kind\":\"dev\",\"name\":\"csv\",\"req\":\"^1\"},{\"default_features\":false,\"name\":\"diesel\",\"optional\":true,\"req\":\"^2.2.3\"},{\"default_features\":false,\"features\":[\"mysql\",\"postgres\"],\"kind\":\"dev\",\"name\":\"diesel\",\"req\":\"^2.2.3\"},{\"default_features\":false,\"kind\":\"dev\",\"name\":\"futures\",\"req\":\"^0.3\"},{\"default_features\":false,\"name\":\"ndarray\",\"optional\":true,\"req\":\"^0.15.6\"},{\"default_features\":false,\"features\":[\"i128\"],\"name\":\"num-traits\",\"req\":\"^0.2\"},{\"default_features\":false,\"kind\":\"dev\",\"name\":\"postgres\",\"req\":\"^0.19\"},{\"default_features\":false,\"name\":\"postgres-types\",\"optional\":true,\"req\":\"^0.2\"},{\"default_features\":false,\"features\":[\"std\"],\"name\":\"proptest\",\"optional\":true,\"req\":\"^1.0\"},{\"default_features\":false,\"name\":\"rand\",\"optional\":true,\"req\":\"^0.8\"},{\"default_features\":false,\"features\":[\"getrandom\"],\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.8\"},{\"default_features\":false,\"name\":\"rand-0_9\",\"optional\":true,\"package\":\"rand\",\"req\":\"^0.9\"},{\"default_features\":false,\"features\":[\"thread_rng\"],\"kind\":\"dev\",\"name\":\"rand-0_9\",\"package\":\"rand\",\"req\":\"^0.9\"},{\"default_features\":false,\"features\":[\"size_32\",\"std\"],\"name\":\"rkyv\",\"optional\":true,\"req\":\"^0.7.42\"},{\"kind\":\"dev\",\"name\":\"rkyv-0_8\",\"package\":\"rkyv\",\"req\":\"^0.8\"},{\"default_features\":false,\"name\":\"rocket\",\"optional\":true,\"req\":\"^0.5.0-rc.3\"},{\"default_features\":false,\"name\":\"rust_decimal_macros\",\"optional\":true,\"req\":\"^1\"},{\"default_features\":false,\"name\":\"serde\",\"optional\":true,\"req\":\"^1.0\"},{\"default_features\":false,\"features\":[\"derive\"],\"kind\":\"dev\",\"name\":\"serde\",\"req\":\"^1.0\"},{\"default_features\":false,\"name\":\"serde_json\",\"optional\":true,\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"serde_json\",\"req\":\"^1.0\"},{\"default_features\":false,\"features\":[\"macros\",\"rt-multi-thread\",\"test-util\"],\"kind\":\"dev\",\"name\":\"tokio\",\"req\":\"^1.0\"},{\"default_features\":false,\"name\":\"tokio-postgres\",\"optional\":true,\"req\":\"^0.7\"},{\"default_features\":false,\"kind\":\"dev\",\"name\":\"tokio-postgres\",\"req\":\"^0.7\"},{\"default_features\":false,\"features\":[\"html_root_url_updated\",\"markdown_deps_updated\"],\"kind\":\"dev\",\"name\":\"version-sync\",\"req\":\"^0.9\"}],\"features\":{\"align16\":[],\"borsh\":[\"dep:borsh\",\"std\"],\"c-repr\":[],\"db-diesel-mysql\":[\"diesel/mysql_backend\",\"std\"],\"db-diesel-postgres\":[\"diesel/postgres_backend\",\"std\"],\"db-diesel2-mysql\":[\"db-diesel-mysql\"],\"db-diesel2-postgres\":[\"db-diesel-postgres\"],\"db-postgres\":[\"dep:bytes\",\"dep:postgres-types\",\"std\"],\"db-tokio-postgres\":[\"dep:bytes\",\"dep:postgres-types\",\"std\"],\"default\":[\"serde\",\"std\"],\"legacy-ops\":[],\"macros\":[\"dep:rust_decimal_macros\"],\"maths\":[],\"maths-nopanic\":[\"maths\"],\"ndarray\":[\"dep:ndarray\"],\"proptest\":[\"dep:proptest\"],\"rand\":[\"dep:rand\"],\"rkyv\":[\"dep:rkyv\"],\"rkyv-safe\":[\"rkyv/validation\"],\"rocket-traits\":[\"dep:rocket\",\"std\"],\"rust-fuzz\":[\"dep:arbitrary\"],\"serde\":[\"dep:serde\"],\"serde-arbitrary-precision\":[\"serde-with-arbitrary-precision\"],\"serde-bincode\":[\"serde-str\"],\"serde-float\":[\"serde-with-float\"],\"serde-str\":[\"serde-with-str\"],\"serde-with-arbitrary-precision\":[\"serde\",\"serde_json/arbitrary_precision\",\"serde_json/std\"],\"serde-with-float\":[\"serde\"],\"serde-with-str\":[\"serde\"],\"std\":[\"arrayvec/std\",\"borsh?/std\",\"bytes?/std\",\"rand?/std\",\"rkyv?/std\",\"serde?/std\",\"serde_json?/std\"],\"tokio-pg\":[\"db-tokio-postgres\"]}}", + "rustc-demangle_0.1.27": "{\"dependencies\":[{\"name\":\"core\",\"optional\":true,\"package\":\"rustc-std-workspace-core\",\"req\":\"^1.0.0\"}],\"features\":{\"compiler_builtins\":[],\"rustc-dep-of-std\":[\"core\"],\"std\":[]}}", "rustc_version_0.4.1": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"doc-comment\",\"req\":\"^0.3\"},{\"name\":\"semver\",\"req\":\"^1.0\"}],\"features\":{}}", "rustc_version_runtime_0.3.0": "{\"dependencies\":[{\"name\":\"rustc_version\",\"req\":\"^0.4.0\"},{\"kind\":\"build\",\"name\":\"rustc_version\",\"req\":\"^0.4.0\"},{\"name\":\"semver\",\"req\":\"^1.0\"},{\"kind\":\"build\",\"name\":\"semver\",\"req\":\"^1.0\"}],\"features\":{}}", "rustix_1.1.2": "{\"dependencies\":[{\"default_features\":false,\"name\":\"bitflags\",\"req\":\"^2.4.0\"},{\"name\":\"core\",\"optional\":true,\"package\":\"rustc-std-workspace-core\",\"req\":\"^1.0.0\"},{\"kind\":\"dev\",\"name\":\"criterion\",\"req\":\"^0.4\",\"target\":\"cfg(all(criterion, not(any(target_os = \\\"emscripten\\\", target_os = \\\"wasi\\\"))))\"},{\"kind\":\"dev\",\"name\":\"flate2\",\"req\":\"^1.0\"},{\"default_features\":false,\"name\":\"libc\",\"req\":\"^0.2.171\",\"target\":\"cfg(all(not(windows), any(rustix_use_libc, miri, not(all(target_os = \\\"linux\\\", any(target_endian = \\\"little\\\", any(target_arch = \\\"s390x\\\", target_arch = \\\"powerpc\\\")), any(target_arch = \\\"arm\\\", all(target_arch = \\\"aarch64\\\", target_pointer_width = \\\"64\\\"), target_arch = \\\"riscv64\\\", all(rustix_use_experimental_asm, target_arch = \\\"powerpc\\\"), all(rustix_use_experimental_asm, target_arch = \\\"powerpc64\\\"), all(rustix_use_experimental_asm, target_arch = \\\"s390x\\\"), all(rustix_use_experimental_asm, target_arch = \\\"mips\\\"), all(rustix_use_experimental_asm, target_arch = \\\"mips32r6\\\"), all(rustix_use_experimental_asm, target_arch = \\\"mips64\\\"), all(rustix_use_experimental_asm, target_arch = \\\"mips64r6\\\"), target_arch = \\\"x86\\\", all(target_arch = \\\"x86_64\\\", target_pointer_width = \\\"64\\\")))))))\"},{\"default_features\":false,\"name\":\"libc\",\"optional\":true,\"req\":\"^0.2.171\",\"target\":\"cfg(all(not(rustix_use_libc), not(miri), target_os = \\\"linux\\\", any(target_endian = \\\"little\\\", any(target_arch = \\\"s390x\\\", target_arch = \\\"powerpc\\\")), any(target_arch = \\\"arm\\\", all(target_arch = \\\"aarch64\\\", target_pointer_width = \\\"64\\\"), target_arch = \\\"riscv64\\\", all(rustix_use_experimental_asm, target_arch = \\\"powerpc\\\"), all(rustix_use_experimental_asm, target_arch = \\\"powerpc64\\\"), all(rustix_use_experimental_asm, target_arch = \\\"s390x\\\"), all(rustix_use_experimental_asm, target_arch = \\\"mips\\\"), all(rustix_use_experimental_asm, target_arch = \\\"mips32r6\\\"), all(rustix_use_experimental_asm, target_arch = \\\"mips64\\\"), all(rustix_use_experimental_asm, target_arch = \\\"mips64r6\\\"), target_arch = \\\"x86\\\", all(target_arch = \\\"x86_64\\\", target_pointer_width = \\\"64\\\"))))\"},{\"kind\":\"dev\",\"name\":\"libc\",\"req\":\"^0.2.171\"},{\"default_features\":false,\"name\":\"libc_errno\",\"package\":\"errno\",\"req\":\"^0.3.10\",\"target\":\"cfg(all(not(windows), any(rustix_use_libc, miri, not(all(target_os = \\\"linux\\\", any(target_endian = \\\"little\\\", any(target_arch = \\\"s390x\\\", target_arch = \\\"powerpc\\\")), any(target_arch = \\\"arm\\\", all(target_arch = \\\"aarch64\\\", target_pointer_width = \\\"64\\\"), target_arch = \\\"riscv64\\\", all(rustix_use_experimental_asm, target_arch = \\\"powerpc\\\"), all(rustix_use_experimental_asm, target_arch = \\\"powerpc64\\\"), all(rustix_use_experimental_asm, target_arch = \\\"s390x\\\"), all(rustix_use_experimental_asm, target_arch = \\\"mips\\\"), all(rustix_use_experimental_asm, target_arch = \\\"mips32r6\\\"), all(rustix_use_experimental_asm, target_arch = \\\"mips64\\\"), all(rustix_use_experimental_asm, target_arch = \\\"mips64r6\\\"), target_arch = \\\"x86\\\", all(target_arch = \\\"x86_64\\\", target_pointer_width = \\\"64\\\")))))))\"},{\"default_features\":false,\"name\":\"libc_errno\",\"package\":\"errno\",\"req\":\"^0.3.10\",\"target\":\"cfg(windows)\"},{\"default_features\":false,\"name\":\"libc_errno\",\"optional\":true,\"package\":\"errno\",\"req\":\"^0.3.10\",\"target\":\"cfg(all(not(rustix_use_libc), not(miri), target_os = \\\"linux\\\", any(target_endian = \\\"little\\\", any(target_arch = \\\"s390x\\\", target_arch = \\\"powerpc\\\")), any(target_arch = \\\"arm\\\", all(target_arch = \\\"aarch64\\\", target_pointer_width = \\\"64\\\"), target_arch = \\\"riscv64\\\", all(rustix_use_experimental_asm, target_arch = \\\"powerpc\\\"), all(rustix_use_experimental_asm, target_arch = \\\"powerpc64\\\"), all(rustix_use_experimental_asm, target_arch = \\\"s390x\\\"), all(rustix_use_experimental_asm, target_arch = \\\"mips\\\"), all(rustix_use_experimental_asm, target_arch = \\\"mips32r6\\\"), all(rustix_use_experimental_asm, target_arch = \\\"mips64\\\"), all(rustix_use_experimental_asm, target_arch = \\\"mips64r6\\\"), target_arch = \\\"x86\\\", all(target_arch = \\\"x86_64\\\", target_pointer_width = \\\"64\\\"))))\"},{\"default_features\":false,\"kind\":\"dev\",\"name\":\"libc_errno\",\"package\":\"errno\",\"req\":\"^0.3.10\"},{\"default_features\":false,\"features\":[\"general\",\"ioctl\",\"no_std\"],\"name\":\"linux-raw-sys\",\"req\":\"^0.11.0\",\"target\":\"cfg(all(any(target_os = \\\"linux\\\"), any(rustix_use_libc, miri, not(all(target_os = \\\"linux\\\", any(target_endian = \\\"little\\\", any(target_arch = \\\"s390x\\\", target_arch = \\\"powerpc\\\")), any(target_arch = \\\"arm\\\", all(target_arch = \\\"aarch64\\\", target_pointer_width = \\\"64\\\"), target_arch = \\\"riscv64\\\", all(rustix_use_experimental_asm, target_arch = \\\"powerpc\\\"), all(rustix_use_experimental_asm, target_arch = \\\"powerpc64\\\"), all(rustix_use_experimental_asm, target_arch = \\\"s390x\\\"), all(rustix_use_experimental_asm, target_arch = \\\"mips\\\"), all(rustix_use_experimental_asm, target_arch = \\\"mips32r6\\\"), all(rustix_use_experimental_asm, target_arch = \\\"mips64\\\"), all(rustix_use_experimental_asm, target_arch = \\\"mips64r6\\\"), target_arch = \\\"x86\\\", all(target_arch = \\\"x86_64\\\", target_pointer_width = \\\"64\\\")))))))\"},{\"default_features\":false,\"features\":[\"auxvec\",\"general\",\"errno\",\"ioctl\",\"no_std\",\"elf\"],\"name\":\"linux-raw-sys\",\"req\":\"^0.11.0\",\"target\":\"cfg(all(not(rustix_use_libc), not(miri), target_os = \\\"linux\\\", any(target_endian = \\\"little\\\", any(target_arch = \\\"s390x\\\", target_arch = \\\"powerpc\\\")), any(target_arch = \\\"arm\\\", all(target_arch = \\\"aarch64\\\", target_pointer_width = \\\"64\\\"), target_arch = \\\"riscv64\\\", all(rustix_use_experimental_asm, target_arch = \\\"powerpc\\\"), all(rustix_use_experimental_asm, target_arch = \\\"powerpc64\\\"), all(rustix_use_experimental_asm, target_arch = \\\"s390x\\\"), all(rustix_use_experimental_asm, target_arch = \\\"mips\\\"), all(rustix_use_experimental_asm, target_arch = \\\"mips32r6\\\"), all(rustix_use_experimental_asm, target_arch = \\\"mips64\\\"), all(rustix_use_experimental_asm, target_arch = \\\"mips64r6\\\"), target_arch = \\\"x86\\\", all(target_arch = \\\"x86_64\\\", target_pointer_width = \\\"64\\\"))))\"},{\"kind\":\"dev\",\"name\":\"memoffset\",\"req\":\"^0.9.0\"},{\"kind\":\"dev\",\"name\":\"once_cell\",\"req\":\"^1.20.3\",\"target\":\"cfg(windows)\"},{\"name\":\"rustc-std-workspace-alloc\",\"optional\":true,\"req\":\"^1.0.0\"},{\"kind\":\"dev\",\"name\":\"serial_test\",\"req\":\"^2.0.0\"},{\"kind\":\"dev\",\"name\":\"static_assertions\",\"req\":\"^1.1.0\"},{\"kind\":\"dev\",\"name\":\"tempfile\",\"req\":\"^3.5.0\"},{\"features\":[\"Win32_Foundation\",\"Win32_Networking_WinSock\"],\"name\":\"windows-sys\",\"req\":\">=0.52, <0.62\",\"target\":\"cfg(windows)\"}],\"features\":{\"all-apis\":[\"event\",\"fs\",\"io_uring\",\"mm\",\"mount\",\"net\",\"param\",\"pipe\",\"process\",\"pty\",\"rand\",\"runtime\",\"shm\",\"stdio\",\"system\",\"termios\",\"thread\",\"time\"],\"alloc\":[],\"default\":[\"std\"],\"event\":[],\"fs\":[],\"io_uring\":[\"event\",\"fs\",\"net\",\"thread\",\"linux-raw-sys/io_uring\"],\"linux_4_11\":[],\"linux_5_1\":[\"linux_4_11\"],\"linux_5_11\":[\"linux_5_1\"],\"linux_latest\":[\"linux_5_11\"],\"mm\":[],\"mount\":[],\"net\":[\"linux-raw-sys/net\",\"linux-raw-sys/netlink\",\"linux-raw-sys/if_ether\",\"linux-raw-sys/xdp\"],\"param\":[],\"pipe\":[],\"process\":[\"linux-raw-sys/prctl\"],\"pty\":[\"fs\"],\"rand\":[],\"runtime\":[\"linux-raw-sys/prctl\"],\"rustc-dep-of-std\":[\"core\",\"rustc-std-workspace-alloc\",\"linux-raw-sys/rustc-dep-of-std\",\"bitflags/rustc-dep-of-std\"],\"shm\":[\"fs\"],\"std\":[\"bitflags/std\",\"alloc\",\"libc?/std\",\"libc_errno?/std\"],\"stdio\":[],\"system\":[\"linux-raw-sys/system\"],\"termios\":[],\"thread\":[\"linux-raw-sys/prctl\"],\"time\":[],\"try_close\":[],\"use-explicitly-provided-auxv\":[],\"use-libc\":[\"libc_errno\",\"libc\"],\"use-libc-auxv\":[]}}", "rustix_1.1.4": "{\"dependencies\":[{\"default_features\":false,\"name\":\"bitflags\",\"req\":\"^2.4.0\"},{\"name\":\"core\",\"optional\":true,\"package\":\"rustc-std-workspace-core\",\"req\":\"^1.0.0\"},{\"kind\":\"dev\",\"name\":\"criterion\",\"req\":\"^0.4\",\"target\":\"cfg(all(criterion, not(any(target_os = \\\"emscripten\\\", target_os = \\\"wasi\\\"))))\"},{\"kind\":\"dev\",\"name\":\"flate2\",\"req\":\"^1.0\"},{\"default_features\":false,\"name\":\"libc\",\"req\":\"^0.2.182\",\"target\":\"cfg(all(not(windows), any(rustix_use_libc, miri, not(all(target_os = \\\"linux\\\", any(target_endian = \\\"little\\\", any(target_arch = \\\"s390x\\\", target_arch = \\\"powerpc\\\")), any(target_arch = \\\"arm\\\", all(target_arch = \\\"aarch64\\\", target_pointer_width = \\\"64\\\"), target_arch = \\\"riscv64\\\", all(rustix_use_experimental_asm, target_arch = \\\"powerpc\\\"), all(rustix_use_experimental_asm, target_arch = \\\"powerpc64\\\"), all(rustix_use_experimental_asm, target_arch = \\\"s390x\\\"), all(rustix_use_experimental_asm, target_arch = \\\"mips\\\"), all(rustix_use_experimental_asm, target_arch = \\\"mips32r6\\\"), all(rustix_use_experimental_asm, target_arch = \\\"mips64\\\"), all(rustix_use_experimental_asm, target_arch = \\\"mips64r6\\\"), target_arch = \\\"x86\\\", all(target_arch = \\\"x86_64\\\", target_pointer_width = \\\"64\\\")))))))\"},{\"default_features\":false,\"name\":\"libc\",\"optional\":true,\"req\":\"^0.2.182\",\"target\":\"cfg(all(not(rustix_use_libc), not(miri), target_os = \\\"linux\\\", any(target_endian = \\\"little\\\", any(target_arch = \\\"s390x\\\", target_arch = \\\"powerpc\\\")), any(target_arch = \\\"arm\\\", all(target_arch = \\\"aarch64\\\", target_pointer_width = \\\"64\\\"), target_arch = \\\"riscv64\\\", all(rustix_use_experimental_asm, target_arch = \\\"powerpc\\\"), all(rustix_use_experimental_asm, target_arch = \\\"powerpc64\\\"), all(rustix_use_experimental_asm, target_arch = \\\"s390x\\\"), all(rustix_use_experimental_asm, target_arch = \\\"mips\\\"), all(rustix_use_experimental_asm, target_arch = \\\"mips32r6\\\"), all(rustix_use_experimental_asm, target_arch = \\\"mips64\\\"), all(rustix_use_experimental_asm, target_arch = \\\"mips64r6\\\"), target_arch = \\\"x86\\\", all(target_arch = \\\"x86_64\\\", target_pointer_width = \\\"64\\\"))))\"},{\"kind\":\"dev\",\"name\":\"libc\",\"req\":\"^0.2.171\"},{\"default_features\":false,\"name\":\"libc_errno\",\"package\":\"errno\",\"req\":\"^0.3.10\",\"target\":\"cfg(all(not(windows), any(rustix_use_libc, miri, not(all(target_os = \\\"linux\\\", any(target_endian = \\\"little\\\", any(target_arch = \\\"s390x\\\", target_arch = \\\"powerpc\\\")), any(target_arch = \\\"arm\\\", all(target_arch = \\\"aarch64\\\", target_pointer_width = \\\"64\\\"), target_arch = \\\"riscv64\\\", all(rustix_use_experimental_asm, target_arch = \\\"powerpc\\\"), all(rustix_use_experimental_asm, target_arch = \\\"powerpc64\\\"), all(rustix_use_experimental_asm, target_arch = \\\"s390x\\\"), all(rustix_use_experimental_asm, target_arch = \\\"mips\\\"), all(rustix_use_experimental_asm, target_arch = \\\"mips32r6\\\"), all(rustix_use_experimental_asm, target_arch = \\\"mips64\\\"), all(rustix_use_experimental_asm, target_arch = \\\"mips64r6\\\"), target_arch = \\\"x86\\\", all(target_arch = \\\"x86_64\\\", target_pointer_width = \\\"64\\\")))))))\"},{\"default_features\":false,\"name\":\"libc_errno\",\"package\":\"errno\",\"req\":\"^0.3.10\",\"target\":\"cfg(windows)\"},{\"default_features\":false,\"name\":\"libc_errno\",\"optional\":true,\"package\":\"errno\",\"req\":\"^0.3.10\",\"target\":\"cfg(all(not(rustix_use_libc), not(miri), target_os = \\\"linux\\\", any(target_endian = \\\"little\\\", any(target_arch = \\\"s390x\\\", target_arch = \\\"powerpc\\\")), any(target_arch = \\\"arm\\\", all(target_arch = \\\"aarch64\\\", target_pointer_width = \\\"64\\\"), target_arch = \\\"riscv64\\\", all(rustix_use_experimental_asm, target_arch = \\\"powerpc\\\"), all(rustix_use_experimental_asm, target_arch = \\\"powerpc64\\\"), all(rustix_use_experimental_asm, target_arch = \\\"s390x\\\"), all(rustix_use_experimental_asm, target_arch = \\\"mips\\\"), all(rustix_use_experimental_asm, target_arch = \\\"mips32r6\\\"), all(rustix_use_experimental_asm, target_arch = \\\"mips64\\\"), all(rustix_use_experimental_asm, target_arch = \\\"mips64r6\\\"), target_arch = \\\"x86\\\", all(target_arch = \\\"x86_64\\\", target_pointer_width = \\\"64\\\"))))\"},{\"default_features\":false,\"kind\":\"dev\",\"name\":\"libc_errno\",\"package\":\"errno\",\"req\":\"^0.3.10\"},{\"default_features\":false,\"features\":[\"general\",\"ioctl\",\"no_std\"],\"name\":\"linux-raw-sys\",\"req\":\"^0.12\",\"target\":\"cfg(all(any(target_os = \\\"linux\\\", target_os = \\\"android\\\"), any(rustix_use_libc, miri, not(all(target_os = \\\"linux\\\", any(target_endian = \\\"little\\\", any(target_arch = \\\"s390x\\\", target_arch = \\\"powerpc\\\")), any(target_arch = \\\"arm\\\", all(target_arch = \\\"aarch64\\\", target_pointer_width = \\\"64\\\"), target_arch = \\\"riscv64\\\", all(rustix_use_experimental_asm, target_arch = \\\"powerpc\\\"), all(rustix_use_experimental_asm, target_arch = \\\"powerpc64\\\"), all(rustix_use_experimental_asm, target_arch = \\\"s390x\\\"), all(rustix_use_experimental_asm, target_arch = \\\"mips\\\"), all(rustix_use_experimental_asm, target_arch = \\\"mips32r6\\\"), all(rustix_use_experimental_asm, target_arch = \\\"mips64\\\"), all(rustix_use_experimental_asm, target_arch = \\\"mips64r6\\\"), target_arch = \\\"x86\\\", all(target_arch = \\\"x86_64\\\", target_pointer_width = \\\"64\\\")))))))\"},{\"default_features\":false,\"features\":[\"auxvec\",\"general\",\"errno\",\"ioctl\",\"no_std\",\"elf\"],\"name\":\"linux-raw-sys\",\"req\":\"^0.12\",\"target\":\"cfg(all(not(rustix_use_libc), not(miri), target_os = \\\"linux\\\", any(target_endian = \\\"little\\\", any(target_arch = \\\"s390x\\\", target_arch = \\\"powerpc\\\")), any(target_arch = \\\"arm\\\", all(target_arch = \\\"aarch64\\\", target_pointer_width = \\\"64\\\"), target_arch = \\\"riscv64\\\", all(rustix_use_experimental_asm, target_arch = \\\"powerpc\\\"), all(rustix_use_experimental_asm, target_arch = \\\"powerpc64\\\"), all(rustix_use_experimental_asm, target_arch = \\\"s390x\\\"), all(rustix_use_experimental_asm, target_arch = \\\"mips\\\"), all(rustix_use_experimental_asm, target_arch = \\\"mips32r6\\\"), all(rustix_use_experimental_asm, target_arch = \\\"mips64\\\"), all(rustix_use_experimental_asm, target_arch = \\\"mips64r6\\\"), target_arch = \\\"x86\\\", all(target_arch = \\\"x86_64\\\", target_pointer_width = \\\"64\\\"))))\"},{\"kind\":\"dev\",\"name\":\"memoffset\",\"req\":\"^0.9.0\"},{\"kind\":\"dev\",\"name\":\"once_cell\",\"req\":\"^1.20.3\",\"target\":\"cfg(windows)\"},{\"name\":\"rustc-std-workspace-alloc\",\"optional\":true,\"req\":\"^1.0.0\"},{\"kind\":\"dev\",\"name\":\"serial_test\",\"req\":\"^2.0.0\"},{\"kind\":\"dev\",\"name\":\"static_assertions\",\"req\":\"^1.1.0\"},{\"kind\":\"dev\",\"name\":\"tempfile\",\"req\":\"^3.5.0\"},{\"features\":[\"Win32_Foundation\",\"Win32_Networking_WinSock\"],\"name\":\"windows-sys\",\"req\":\">=0.52, <0.62\",\"target\":\"cfg(windows)\"}],\"features\":{\"all-apis\":[\"event\",\"fs\",\"io_uring\",\"mm\",\"mount\",\"net\",\"param\",\"pipe\",\"process\",\"pty\",\"rand\",\"runtime\",\"shm\",\"stdio\",\"system\",\"termios\",\"thread\",\"time\"],\"alloc\":[],\"default\":[\"std\"],\"event\":[],\"fs\":[],\"io_uring\":[\"event\",\"fs\",\"net\",\"thread\",\"linux-raw-sys/io_uring\"],\"linux_4_11\":[],\"linux_5_1\":[\"linux_4_11\"],\"linux_5_11\":[\"linux_5_1\"],\"linux_latest\":[\"linux_5_11\"],\"mm\":[],\"mount\":[],\"net\":[\"linux-raw-sys/net\",\"linux-raw-sys/netlink\",\"linux-raw-sys/if_ether\",\"linux-raw-sys/xdp\"],\"param\":[],\"pipe\":[],\"process\":[\"linux-raw-sys/prctl\"],\"pty\":[\"fs\"],\"rand\":[],\"runtime\":[\"linux-raw-sys/prctl\"],\"rustc-dep-of-std\":[\"core\",\"rustc-std-workspace-alloc\",\"linux-raw-sys/rustc-dep-of-std\",\"bitflags/rustc-dep-of-std\"],\"shm\":[\"fs\"],\"std\":[\"bitflags/std\",\"alloc\",\"libc?/std\",\"libc_errno?/std\"],\"stdio\":[],\"system\":[\"linux-raw-sys/system\"],\"termios\":[],\"thread\":[\"linux-raw-sys/prctl\"],\"time\":[],\"try_close\":[],\"use-explicitly-provided-auxv\":[],\"use-libc\":[\"libc_errno\",\"libc\"],\"use-libc-auxv\":[]}}", "rustls-native-certs_0.8.2": "{\"dependencies\":[{\"name\":\"openssl-probe\",\"req\":\"^0.1.6\",\"target\":\"cfg(all(unix, not(target_os = \\\"macos\\\")))\"},{\"features\":[\"std\"],\"name\":\"pki-types\",\"package\":\"rustls-pki-types\",\"req\":\"^1.10\"},{\"kind\":\"dev\",\"name\":\"ring\",\"req\":\"^0.17\"},{\"kind\":\"dev\",\"name\":\"rustls\",\"req\":\"^0.23\"},{\"kind\":\"dev\",\"name\":\"rustls-webpki\",\"req\":\"^0.103\"},{\"name\":\"schannel\",\"req\":\"^0.1\",\"target\":\"cfg(windows)\"},{\"name\":\"security-framework\",\"req\":\"^3\",\"target\":\"cfg(target_os = \\\"macos\\\")\"},{\"kind\":\"dev\",\"name\":\"serial_test\",\"req\":\"^3\"},{\"kind\":\"dev\",\"name\":\"tempfile\",\"req\":\"^3.5\"},{\"kind\":\"dev\",\"name\":\"untrusted\",\"req\":\"^0.9\"},{\"kind\":\"dev\",\"name\":\"webpki-roots\",\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"x509-parser\",\"req\":\"^0.17\"}],\"features\":{}}", "rustls-pki-types_1.13.1": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"crabgrind\",\"req\":\"=0.1.9\",\"target\":\"cfg(all(target_os = \\\"linux\\\", target_arch = \\\"x86_64\\\"))\"},{\"name\":\"web-time\",\"optional\":true,\"req\":\"^1\",\"target\":\"cfg(all(target_family = \\\"wasm\\\", target_os = \\\"unknown\\\"))\"},{\"name\":\"zeroize\",\"optional\":true,\"req\":\"^1\"}],\"features\":{\"alloc\":[\"dep:zeroize\"],\"default\":[\"alloc\"],\"std\":[\"alloc\"],\"web\":[\"web-time\"]}}", + "rustls-pki-types_1.14.1": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"crabgrind\",\"req\":\"=0.1.9\",\"target\":\"cfg(all(target_os = \\\"linux\\\", target_arch = \\\"x86_64\\\"))\"},{\"name\":\"web-time\",\"optional\":true,\"req\":\"^1\",\"target\":\"cfg(all(target_family = \\\"wasm\\\", target_os = \\\"unknown\\\"))\"},{\"name\":\"zeroize\",\"optional\":true,\"req\":\"^1\"}],\"features\":{\"alloc\":[\"dep:zeroize\"],\"default\":[\"alloc\"],\"std\":[\"alloc\"],\"web\":[\"web-time\"]}}", "rustls-platform-verifier-android_0.1.1": "{\"dependencies\":[],\"features\":{}}", "rustls-platform-verifier_0.6.2": "{\"dependencies\":[{\"name\":\"android_logger\",\"optional\":true,\"req\":\"^0.15\",\"target\":\"cfg(target_os = \\\"android\\\")\"},{\"name\":\"base64\",\"optional\":true,\"req\":\"^0.22\"},{\"name\":\"core-foundation\",\"req\":\"^0.10\",\"target\":\"cfg(any(target_vendor = \\\"apple\\\"))\"},{\"name\":\"core-foundation-sys\",\"req\":\"^0.8\",\"target\":\"cfg(any(target_vendor = \\\"apple\\\"))\"},{\"default_features\":false,\"name\":\"jni\",\"req\":\"^0.21\",\"target\":\"cfg(target_os = \\\"android\\\")\"},{\"default_features\":false,\"name\":\"jni\",\"optional\":true,\"req\":\"^0.21\"},{\"name\":\"log\",\"req\":\"^0.4\"},{\"name\":\"once_cell\",\"req\":\"^1.9\",\"target\":\"cfg(target_os = \\\"android\\\")\"},{\"name\":\"once_cell\",\"optional\":true,\"req\":\"^1.9\"},{\"default_features\":false,\"features\":[\"std\"],\"name\":\"rustls\",\"req\":\"^0.23.27\"},{\"default_features\":false,\"features\":[\"ring\"],\"kind\":\"dev\",\"name\":\"rustls\",\"req\":\"^0.23\"},{\"name\":\"rustls-native-certs\",\"req\":\"^0.8\",\"target\":\"cfg(all(unix, not(target_os = \\\"android\\\"), not(target_vendor = \\\"apple\\\"), not(target_arch = \\\"wasm32\\\")))\"},{\"name\":\"rustls-platform-verifier-android\",\"req\":\"^0.1.0\",\"target\":\"cfg(target_os = \\\"android\\\")\"},{\"name\":\"security-framework\",\"req\":\"^3.5.0\",\"target\":\"cfg(any(target_vendor = \\\"apple\\\"))\"},{\"name\":\"security-framework-sys\",\"req\":\"^2.15\",\"target\":\"cfg(any(target_vendor = \\\"apple\\\"))\"},{\"default_features\":false,\"name\":\"webpki\",\"package\":\"rustls-webpki\",\"req\":\"^0.103\",\"target\":\"cfg(all(unix, not(target_os = \\\"android\\\"), not(target_vendor = \\\"apple\\\"), not(target_arch = \\\"wasm32\\\")))\"},{\"default_features\":false,\"name\":\"webpki\",\"package\":\"rustls-webpki\",\"req\":\"^0.103\",\"target\":\"cfg(target_arch = \\\"wasm32\\\")\"},{\"default_features\":false,\"name\":\"webpki\",\"package\":\"rustls-webpki\",\"req\":\"^0.103\",\"target\":\"cfg(target_os = \\\"android\\\")\"},{\"name\":\"webpki-root-certs\",\"req\":\"^1\",\"target\":\"cfg(target_arch = \\\"wasm32\\\")\"},{\"kind\":\"dev\",\"name\":\"webpki-root-certs\",\"req\":\"^1\"},{\"default_features\":false,\"features\":[\"Win32_Foundation\",\"Win32_Security_Cryptography\"],\"name\":\"windows-sys\",\"req\":\">=0.52.0, <0.62.0\",\"target\":\"cfg(windows)\"}],\"features\":{\"cert-logging\":[\"base64\"],\"dbg\":[],\"docsrs\":[\"jni\",\"once_cell\"],\"ffi-testing\":[\"android_logger\",\"rustls/ring\"]}}", "rustls-webpki_0.103.13": "{\"dependencies\":[{\"default_features\":false,\"name\":\"aws-lc-rs\",\"optional\":true,\"req\":\"^1.14\"},{\"kind\":\"dev\",\"name\":\"base64\",\"req\":\"^0.22\"},{\"kind\":\"dev\",\"name\":\"bencher\",\"req\":\"^0.1.5\"},{\"kind\":\"dev\",\"name\":\"bzip2\",\"req\":\"^0.6\"},{\"kind\":\"dev\",\"name\":\"once_cell\",\"req\":\"^1.17.2\"},{\"default_features\":false,\"name\":\"pki-types\",\"package\":\"rustls-pki-types\",\"req\":\"^1.12\"},{\"default_features\":false,\"features\":[\"aws_lc_rs\"],\"kind\":\"dev\",\"name\":\"rcgen\",\"req\":\"^0.14.2\"},{\"default_features\":false,\"name\":\"ring\",\"optional\":true,\"req\":\"^0.17\"},{\"features\":[\"derive\"],\"kind\":\"dev\",\"name\":\"serde\",\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"serde_json\",\"req\":\"^1.0\"},{\"name\":\"untrusted\",\"req\":\"^0.9\"},{\"kind\":\"dev\",\"name\":\"x509-parser\",\"req\":\"^0.18.1\"}],\"features\":{\"alloc\":[\"ring?/alloc\",\"pki-types/alloc\"],\"aws-lc-rs\":[\"dep:aws-lc-rs\",\"aws-lc-rs/aws-lc-sys\",\"aws-lc-rs/prebuilt-nasm\"],\"aws-lc-rs-fips\":[\"dep:aws-lc-rs\",\"aws-lc-rs/fips\"],\"aws-lc-rs-unstable\":[\"aws-lc-rs\",\"aws-lc-rs/unstable\"],\"default\":[\"std\"],\"ring\":[\"dep:ring\"],\"std\":[\"alloc\",\"pki-types/std\"]}}", "rustls_0.23.34": "{\"dependencies\":[{\"default_features\":false,\"name\":\"aws-lc-rs\",\"optional\":true,\"req\":\"^1.14\"},{\"kind\":\"dev\",\"name\":\"base64\",\"req\":\"^0.22\"},{\"kind\":\"dev\",\"name\":\"bencher\",\"req\":\"^0.1.5\"},{\"default_features\":false,\"features\":[\"std\"],\"name\":\"brotli\",\"optional\":true,\"req\":\"^8\"},{\"name\":\"brotli-decompressor\",\"optional\":true,\"req\":\"^5.0.0\"},{\"kind\":\"dev\",\"name\":\"env_logger\",\"req\":\"^0.11\"},{\"default_features\":false,\"features\":[\"default-hasher\",\"inline-more\"],\"name\":\"hashbrown\",\"optional\":true,\"req\":\"^0.15\"},{\"kind\":\"dev\",\"name\":\"hex\",\"req\":\"^0.4\"},{\"name\":\"log\",\"optional\":true,\"req\":\"^0.4.8\"},{\"kind\":\"dev\",\"name\":\"log\",\"req\":\"^0.4.8\"},{\"kind\":\"dev\",\"name\":\"macro_rules_attribute\",\"req\":\"^0.2\"},{\"kind\":\"dev\",\"name\":\"num-bigint\",\"req\":\"^0.4.4\"},{\"default_features\":false,\"features\":[\"alloc\",\"race\"],\"name\":\"once_cell\",\"req\":\"^1.16\"},{\"features\":[\"alloc\"],\"name\":\"pki-types\",\"package\":\"rustls-pki-types\",\"req\":\"^1.12\"},{\"default_features\":false,\"features\":[\"pem\",\"aws_lc_rs\"],\"kind\":\"dev\",\"name\":\"rcgen\",\"req\":\"^0.14\"},{\"name\":\"ring\",\"optional\":true,\"req\":\"^0.17\"},{\"kind\":\"build\",\"name\":\"rustversion\",\"optional\":true,\"req\":\"^1.0.6\"},{\"features\":[\"derive\"],\"kind\":\"dev\",\"name\":\"serde\",\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"serde_json\",\"req\":\"^1\"},{\"default_features\":false,\"name\":\"subtle\",\"req\":\"^2.5.0\"},{\"default_features\":false,\"kind\":\"dev\",\"name\":\"time\",\"req\":\"^0.3.6\"},{\"default_features\":false,\"features\":[\"alloc\"],\"name\":\"webpki\",\"package\":\"rustls-webpki\",\"req\":\"^0.103.5\"},{\"kind\":\"dev\",\"name\":\"webpki-roots\",\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"x509-parser\",\"req\":\"^0.17\"},{\"name\":\"zeroize\",\"req\":\"^1.8\"},{\"name\":\"zlib-rs\",\"optional\":true,\"req\":\"^0.5\"}],\"features\":{\"aws-lc-rs\":[\"aws_lc_rs\"],\"aws_lc_rs\":[\"dep:aws-lc-rs\",\"webpki/aws-lc-rs\",\"aws-lc-rs/aws-lc-sys\",\"aws-lc-rs/prebuilt-nasm\"],\"brotli\":[\"dep:brotli\",\"dep:brotli-decompressor\",\"std\"],\"custom-provider\":[],\"default\":[\"aws_lc_rs\",\"logging\",\"prefer-post-quantum\",\"std\",\"tls12\"],\"fips\":[\"aws_lc_rs\",\"aws-lc-rs?/fips\",\"webpki/aws-lc-rs-fips\"],\"logging\":[\"log\"],\"prefer-post-quantum\":[\"aws_lc_rs\"],\"read_buf\":[\"rustversion\",\"std\"],\"ring\":[\"dep:ring\",\"webpki/ring\"],\"std\":[\"webpki/std\",\"pki-types/std\",\"once_cell/std\"],\"tls12\":[],\"zlib\":[\"dep:zlib-rs\"]}}", + "rustls_0.23.40": "{\"dependencies\":[{\"default_features\":false,\"name\":\"aws-lc-rs\",\"optional\":true,\"req\":\"^1.14\"},{\"kind\":\"dev\",\"name\":\"base64\",\"req\":\"^0.22\"},{\"kind\":\"dev\",\"name\":\"bencher\",\"req\":\"^0.1.5\"},{\"default_features\":false,\"features\":[\"std\"],\"name\":\"brotli\",\"optional\":true,\"req\":\"^8\"},{\"name\":\"brotli-decompressor\",\"optional\":true,\"req\":\"^5.0.0\"},{\"kind\":\"dev\",\"name\":\"env_logger\",\"req\":\"^0.11\"},{\"default_features\":false,\"features\":[\"default-hasher\",\"inline-more\"],\"name\":\"hashbrown\",\"optional\":true,\"req\":\"^0.15\"},{\"kind\":\"dev\",\"name\":\"hex\",\"req\":\"^0.4\"},{\"name\":\"log\",\"optional\":true,\"req\":\"^0.4.8\"},{\"kind\":\"dev\",\"name\":\"log\",\"req\":\"^0.4.8\"},{\"kind\":\"dev\",\"name\":\"macro_rules_attribute\",\"req\":\"^0.2\"},{\"kind\":\"dev\",\"name\":\"num-bigint\",\"req\":\"^0.4.4\"},{\"default_features\":false,\"features\":[\"alloc\",\"race\"],\"name\":\"once_cell\",\"req\":\"^1.16\"},{\"features\":[\"alloc\"],\"name\":\"pki-types\",\"package\":\"rustls-pki-types\",\"req\":\"^1.12\"},{\"default_features\":false,\"features\":[\"pem\",\"aws_lc_rs\"],\"kind\":\"dev\",\"name\":\"rcgen\",\"req\":\"^0.14\"},{\"name\":\"ring\",\"optional\":true,\"req\":\"^0.17\"},{\"kind\":\"build\",\"name\":\"rustversion\",\"optional\":true,\"req\":\"^1.0.6\"},{\"features\":[\"derive\"],\"kind\":\"dev\",\"name\":\"serde\",\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"serde_json\",\"req\":\"^1\"},{\"default_features\":false,\"name\":\"subtle\",\"req\":\"^2.5.0\"},{\"default_features\":false,\"kind\":\"dev\",\"name\":\"time\",\"req\":\"^0.3.6\"},{\"default_features\":false,\"features\":[\"alloc\"],\"name\":\"webpki\",\"package\":\"rustls-webpki\",\"req\":\"^0.103.5\"},{\"kind\":\"dev\",\"name\":\"webpki-roots\",\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"x509-parser\",\"req\":\"^0.17\"},{\"name\":\"zeroize\",\"req\":\"^1.8\"},{\"name\":\"zlib-rs\",\"optional\":true,\"req\":\"^0.6\"}],\"features\":{\"aws-lc-rs\":[\"aws_lc_rs\"],\"aws_lc_rs\":[\"dep:aws-lc-rs\",\"webpki/aws-lc-rs\",\"aws-lc-rs/aws-lc-sys\",\"aws-lc-rs/prebuilt-nasm\"],\"brotli\":[\"dep:brotli\",\"dep:brotli-decompressor\",\"std\"],\"custom-provider\":[],\"default\":[\"aws_lc_rs\",\"logging\",\"prefer-post-quantum\",\"std\",\"tls12\"],\"fips\":[\"aws_lc_rs\",\"aws-lc-rs?/fips\",\"webpki/aws-lc-rs-fips\"],\"logging\":[\"log\"],\"prefer-post-quantum\":[\"aws_lc_rs\"],\"read_buf\":[\"rustversion\",\"std\"],\"ring\":[\"dep:ring\",\"webpki/ring\"],\"std\":[\"webpki/std\",\"pki-types/std\",\"once_cell/std\"],\"tls12\":[],\"zlib\":[\"dep:zlib-rs\"]}}", "rustversion_1.0.22": "{\"dependencies\":[{\"features\":[\"diff\"],\"kind\":\"dev\",\"name\":\"trybuild\",\"req\":\"^1.0.49\"}],\"features\":{}}", "ryu_1.0.23": "{\"dependencies\":[{\"default_features\":false,\"kind\":\"dev\",\"name\":\"criterion\",\"req\":\"^0.8\",\"target\":\"cfg(not(miri))\"},{\"name\":\"no-panic\",\"optional\":true,\"req\":\"^0.1\"},{\"kind\":\"dev\",\"name\":\"num_cpus\",\"req\":\"^1.8\"},{\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.10\"},{\"kind\":\"dev\",\"name\":\"rand_core\",\"req\":\"^0.10\"},{\"kind\":\"dev\",\"name\":\"rand_xorshift\",\"req\":\"^0.5\"}],\"features\":{\"small\":[]}}", + "safemem_0.3.3": "{\"dependencies\":[],\"features\":{\"default\":[\"std\"],\"std\":[]}}", "same-file_1.0.6": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"doc-comment\",\"req\":\"^0.3\"},{\"name\":\"winapi-util\",\"req\":\"^0.1.1\",\"target\":\"cfg(windows)\"}],\"features\":{}}", "schannel_0.1.28": "{\"dependencies\":[{\"features\":[\"Win32_Foundation\",\"Win32_Security_Cryptography\",\"Win32_Security_Authentication_Identity\",\"Win32_Security_Credentials\",\"Win32_System_LibraryLoader\",\"Win32_System_Memory\",\"Win32_System_SystemInformation\"],\"name\":\"windows-sys\",\"req\":\"^0.61\"},{\"features\":[\"Win32_System_SystemInformation\",\"Win32_System_Time\"],\"kind\":\"dev\",\"name\":\"windows-sys\",\"req\":\"^0.61\"}],\"features\":{}}", "schemars_0.9.0": "{\"dependencies\":[{\"default_features\":false,\"name\":\"arrayvec07\",\"optional\":true,\"package\":\"arrayvec\",\"req\":\"^0.7\"},{\"default_features\":false,\"features\":[\"serde\"],\"kind\":\"dev\",\"name\":\"arrayvec07\",\"package\":\"arrayvec\",\"req\":\"^0.7\"},{\"default_features\":false,\"name\":\"bigdecimal04\",\"optional\":true,\"package\":\"bigdecimal\",\"req\":\"^0.4\"},{\"default_features\":false,\"features\":[\"serde\"],\"kind\":\"dev\",\"name\":\"bigdecimal04\",\"package\":\"bigdecimal\",\"req\":\"^0.4\"},{\"default_features\":false,\"name\":\"bytes1\",\"optional\":true,\"package\":\"bytes\",\"req\":\"^1.0\"},{\"default_features\":false,\"features\":[\"serde\"],\"kind\":\"dev\",\"name\":\"bytes1\",\"package\":\"bytes\",\"req\":\"^1.0\"},{\"default_features\":false,\"name\":\"chrono04\",\"optional\":true,\"package\":\"chrono\",\"req\":\"^0.4\"},{\"default_features\":false,\"features\":[\"serde\"],\"kind\":\"dev\",\"name\":\"chrono04\",\"package\":\"chrono\",\"req\":\"^0.4\"},{\"name\":\"dyn-clone\",\"req\":\"^1.0\"},{\"default_features\":false,\"name\":\"either1\",\"optional\":true,\"package\":\"either\",\"req\":\"^1.3\"},{\"default_features\":false,\"features\":[\"serde\"],\"kind\":\"dev\",\"name\":\"either1\",\"package\":\"either\",\"req\":\"^1.3\"},{\"features\":[\"derive\",\"email\",\"regex\",\"url\"],\"kind\":\"dev\",\"name\":\"garde\",\"req\":\"^0.22\"},{\"default_features\":false,\"name\":\"indexmap2\",\"optional\":true,\"package\":\"indexmap\",\"req\":\"^2.0\"},{\"default_features\":false,\"features\":[\"serde\"],\"kind\":\"dev\",\"name\":\"indexmap2\",\"package\":\"indexmap\",\"req\":\"^2.0\"},{\"default_features\":false,\"name\":\"jiff02\",\"optional\":true,\"package\":\"jiff\",\"req\":\"^0.2\"},{\"features\":[\"serde\"],\"kind\":\"dev\",\"name\":\"jiff02\",\"package\":\"jiff\",\"req\":\"^0.2\"},{\"default_features\":false,\"kind\":\"dev\",\"name\":\"jsonschema\",\"req\":\"^0.30\"},{\"kind\":\"dev\",\"name\":\"pretty_assertions\",\"req\":\"^1.2.1\"},{\"name\":\"ref-cast\",\"req\":\"^1.0.22\"},{\"default_features\":false,\"kind\":\"dev\",\"name\":\"regex\",\"req\":\"^1.10.6\"},{\"default_features\":false,\"name\":\"rust_decimal1\",\"optional\":true,\"package\":\"rust_decimal\",\"req\":\"^1\"},{\"default_features\":false,\"features\":[\"serde\"],\"kind\":\"dev\",\"name\":\"rust_decimal1\",\"package\":\"rust_decimal\",\"req\":\"^1\"},{\"name\":\"schemars_derive\",\"optional\":true,\"req\":\"=0.9.0\"},{\"default_features\":false,\"name\":\"semver1\",\"optional\":true,\"package\":\"semver\",\"req\":\"^1.0.9\"},{\"default_features\":false,\"features\":[\"serde\"],\"kind\":\"dev\",\"name\":\"semver1\",\"package\":\"semver\",\"req\":\"^1.0.9\"},{\"default_features\":false,\"features\":[\"alloc\"],\"name\":\"serde\",\"req\":\"^1.0\"},{\"features\":[\"derive\"],\"kind\":\"dev\",\"name\":\"serde\",\"req\":\"^1.0\"},{\"default_features\":false,\"features\":[\"alloc\"],\"name\":\"serde_json\",\"req\":\"^1.0.127\"},{\"kind\":\"dev\",\"name\":\"serde_repr\",\"req\":\"^0.1.19\"},{\"default_features\":false,\"name\":\"smallvec1\",\"optional\":true,\"package\":\"smallvec\",\"req\":\"^1.0\"},{\"default_features\":false,\"features\":[\"serde\"],\"kind\":\"dev\",\"name\":\"smallvec1\",\"package\":\"smallvec\",\"req\":\"^1.0\"},{\"default_features\":false,\"name\":\"smol_str02\",\"optional\":true,\"package\":\"smol_str\",\"req\":\"^0.2.1\"},{\"default_features\":false,\"features\":[\"serde\"],\"kind\":\"dev\",\"name\":\"smol_str02\",\"package\":\"smol_str\",\"req\":\"^0.2.1\"},{\"features\":[\"json\"],\"kind\":\"dev\",\"name\":\"snapbox\",\"req\":\"^0.6.17\"},{\"kind\":\"dev\",\"name\":\"trybuild\",\"req\":\"^1.0\"},{\"default_features\":false,\"name\":\"url2\",\"optional\":true,\"package\":\"url\",\"req\":\"^2.0\"},{\"default_features\":false,\"features\":[\"serde\",\"std\"],\"kind\":\"dev\",\"name\":\"url2\",\"package\":\"url\",\"req\":\"^2.0\"},{\"default_features\":false,\"name\":\"uuid1\",\"optional\":true,\"package\":\"uuid\",\"req\":\"^1.0\"},{\"default_features\":false,\"features\":[\"serde\"],\"kind\":\"dev\",\"name\":\"uuid1\",\"package\":\"uuid\",\"req\":\"^1.0\"},{\"features\":[\"derive\"],\"kind\":\"dev\",\"name\":\"validator\",\"req\":\"^0.20\"}],\"features\":{\"_ui_test\":[],\"default\":[\"derive\",\"std\"],\"derive\":[\"schemars_derive\"],\"preserve_order\":[\"serde_json/preserve_order\"],\"raw_value\":[\"serde_json/raw_value\"],\"std\":[]}}", @@ -1205,6 +1155,7 @@ "security-framework-sys_2.15.0": "{\"dependencies\":[{\"name\":\"core-foundation-sys\",\"req\":\"^0.8.6\"},{\"name\":\"libc\",\"req\":\"^0.2.150\"}],\"features\":{\"OSX_10_10\":[\"OSX_10_9\"],\"OSX_10_11\":[\"OSX_10_10\"],\"OSX_10_12\":[\"OSX_10_11\"],\"OSX_10_13\":[\"OSX_10_12\"],\"OSX_10_14\":[\"OSX_10_13\"],\"OSX_10_15\":[\"OSX_10_14\"],\"OSX_10_9\":[],\"default\":[\"OSX_10_12\"]}}", "security-framework_3.5.1": "{\"dependencies\":[{\"name\":\"bitflags\",\"req\":\"^2.6\"},{\"name\":\"core-foundation\",\"req\":\"^0.10\"},{\"name\":\"core-foundation-sys\",\"req\":\"^0.8.6\"},{\"kind\":\"dev\",\"name\":\"env_logger\",\"req\":\"^0.11\"},{\"kind\":\"dev\",\"name\":\"hex\",\"req\":\"^0.4.3\"},{\"name\":\"libc\",\"req\":\"^0.2.139\"},{\"name\":\"log\",\"optional\":true,\"req\":\"^0.4.20\"},{\"default_features\":false,\"name\":\"security-framework-sys\",\"req\":\"^2.15\"},{\"kind\":\"dev\",\"name\":\"tempfile\",\"req\":\"^3.12.0\"},{\"kind\":\"dev\",\"name\":\"time\",\"req\":\"^0.3.23\"},{\"kind\":\"dev\",\"name\":\"x509-parser\",\"req\":\"^0.16\"}],\"features\":{\"OSX_10_12\":[\"security-framework-sys/OSX_10_12\"],\"OSX_10_13\":[\"OSX_10_12\",\"security-framework-sys/OSX_10_13\",\"alpn\",\"session-tickets\"],\"OSX_10_14\":[\"OSX_10_13\",\"security-framework-sys/OSX_10_14\"],\"OSX_10_15\":[\"OSX_10_14\",\"security-framework-sys/OSX_10_15\"],\"alpn\":[],\"default\":[\"OSX_10_12\"],\"job-bless\":[],\"nightly\":[],\"session-tickets\":[],\"sync-keychain\":[\"OSX_10_13\"]}}", "semver_1.0.27": "{\"dependencies\":[{\"default_features\":false,\"name\":\"serde\",\"optional\":true,\"package\":\"serde_core\",\"req\":\"^1.0.220\"},{\"default_features\":false,\"name\":\"serde\",\"optional\":true,\"req\":\"^1.0.220\",\"target\":\"cfg(any())\"}],\"features\":{\"default\":[\"std\"],\"serde\":[\"dep:serde\"],\"std\":[]}}", + "semver_1.0.28": "{\"dependencies\":[{\"default_features\":false,\"kind\":\"dev\",\"name\":\"criterion\",\"req\":\"^0.8\",\"target\":\"cfg(not(miri))\"},{\"default_features\":false,\"name\":\"serde\",\"optional\":true,\"package\":\"serde_core\",\"req\":\"^1.0.220\"},{\"default_features\":false,\"name\":\"serde\",\"optional\":true,\"req\":\"^1.0.220\",\"target\":\"cfg(any())\"}],\"features\":{\"default\":[\"std\"],\"serde\":[\"dep:serde\"],\"std\":[]}}", "separator_0.4.1": "{\"dependencies\":[],\"features\":{}}", "serde_1.0.228": "{\"dependencies\":[{\"default_features\":false,\"features\":[\"result\"],\"name\":\"serde_core\",\"req\":\"=1.0.228\"},{\"name\":\"serde_derive\",\"optional\":true,\"req\":\"^1\"}],\"features\":{\"alloc\":[\"serde_core/alloc\"],\"default\":[\"std\"],\"derive\":[\"serde_derive\"],\"rc\":[\"serde_core/rc\"],\"std\":[\"serde_core/std\"],\"unstable\":[\"serde_core/unstable\"]}}", "serde_bytes_0.11.19": "{\"dependencies\":[{\"features\":[\"serde\"],\"kind\":\"dev\",\"name\":\"bincode\",\"req\":\"^2\"},{\"default_features\":false,\"name\":\"serde\",\"req\":\"^1.0.220\",\"target\":\"cfg(any())\"},{\"kind\":\"dev\",\"name\":\"serde\",\"req\":\"^1.0.220\"},{\"default_features\":false,\"name\":\"serde_core\",\"req\":\"^1.0.220\"},{\"kind\":\"dev\",\"name\":\"serde_derive\",\"req\":\"^1.0.220\"},{\"kind\":\"dev\",\"name\":\"serde_test\",\"req\":\"^1.0.166\"}],\"features\":{\"alloc\":[\"serde_core/alloc\"],\"default\":[\"std\"],\"std\":[\"serde_core/std\"]}}", @@ -1212,12 +1163,13 @@ "serde_derive_1.0.228": "{\"dependencies\":[{\"default_features\":false,\"features\":[\"proc-macro\"],\"name\":\"proc-macro2\",\"req\":\"^1.0.74\"},{\"default_features\":false,\"features\":[\"proc-macro\"],\"name\":\"quote\",\"req\":\"^1.0.35\"},{\"kind\":\"dev\",\"name\":\"serde\",\"req\":\"^1\"},{\"default_features\":false,\"features\":[\"clone-impls\",\"derive\",\"parsing\",\"printing\",\"proc-macro\"],\"name\":\"syn\",\"req\":\"^2.0.81\"}],\"features\":{\"default\":[],\"deserialize_in_place\":[]}}", "serde_derive_internals_0.29.1": "{\"dependencies\":[{\"default_features\":false,\"name\":\"proc-macro2\",\"req\":\"^1.0.74\"},{\"default_features\":false,\"name\":\"quote\",\"req\":\"^1.0.35\"},{\"default_features\":false,\"features\":[\"clone-impls\",\"derive\",\"parsing\",\"printing\"],\"name\":\"syn\",\"req\":\"^2.0.46\"}],\"features\":{}}", "serde_json5_0.2.1": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"matches\",\"req\":\"^0.1.8\"},{\"name\":\"pest\",\"req\":\"^2.0\"},{\"name\":\"pest_derive\",\"req\":\"^2.0\"},{\"name\":\"serde\",\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"serde_derive\",\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"serde_json\",\"req\":\"^1.0\"}],\"features\":{}}", + "serde_json_1.0.145": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"automod\",\"req\":\"^1.0.11\"},{\"name\":\"indexmap\",\"optional\":true,\"req\":\"^2.2.3\"},{\"kind\":\"dev\",\"name\":\"indoc\",\"req\":\"^2.0.2\"},{\"name\":\"itoa\",\"req\":\"^1.0\"},{\"default_features\":false,\"name\":\"memchr\",\"req\":\"^2\"},{\"kind\":\"dev\",\"name\":\"ref-cast\",\"req\":\"^1.0.18\"},{\"kind\":\"dev\",\"name\":\"rustversion\",\"req\":\"^1.0.13\"},{\"name\":\"ryu\",\"req\":\"^1.0\"},{\"default_features\":false,\"name\":\"serde\",\"req\":\"^1.0.220\",\"target\":\"cfg(any())\"},{\"features\":[\"derive\"],\"kind\":\"dev\",\"name\":\"serde\",\"req\":\"^1.0.194\"},{\"kind\":\"dev\",\"name\":\"serde_bytes\",\"req\":\"^0.11.10\"},{\"default_features\":false,\"name\":\"serde_core\",\"req\":\"^1.0.220\"},{\"kind\":\"dev\",\"name\":\"serde_derive\",\"req\":\"^1.0.166\"},{\"kind\":\"dev\",\"name\":\"serde_stacker\",\"req\":\"^0.1.8\"},{\"features\":[\"diff\"],\"kind\":\"dev\",\"name\":\"trybuild\",\"req\":\"^1.0.108\"}],\"features\":{\"alloc\":[\"serde_core/alloc\"],\"arbitrary_precision\":[],\"default\":[\"std\"],\"float_roundtrip\":[],\"preserve_order\":[\"indexmap\",\"std\"],\"raw_value\":[],\"std\":[\"memchr/std\",\"serde_core/std\"],\"unbounded_depth\":[]}}", "serde_json_1.0.149": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"automod\",\"req\":\"^1.0.11\"},{\"name\":\"indexmap\",\"optional\":true,\"req\":\"^2.2.3\"},{\"kind\":\"dev\",\"name\":\"indoc\",\"req\":\"^2.0.2\"},{\"name\":\"itoa\",\"req\":\"^1.0\"},{\"default_features\":false,\"name\":\"memchr\",\"req\":\"^2\"},{\"kind\":\"dev\",\"name\":\"ref-cast\",\"req\":\"^1.0.18\"},{\"kind\":\"dev\",\"name\":\"rustversion\",\"req\":\"^1.0.13\"},{\"default_features\":false,\"name\":\"serde\",\"req\":\"^1.0.220\",\"target\":\"cfg(any())\"},{\"features\":[\"derive\"],\"kind\":\"dev\",\"name\":\"serde\",\"req\":\"^1.0.194\"},{\"kind\":\"dev\",\"name\":\"serde_bytes\",\"req\":\"^0.11.10\"},{\"default_features\":false,\"name\":\"serde_core\",\"req\":\"^1.0.220\"},{\"kind\":\"dev\",\"name\":\"serde_derive\",\"req\":\"^1.0.166\"},{\"kind\":\"dev\",\"name\":\"serde_stacker\",\"req\":\"^0.1.8\"},{\"features\":[\"diff\"],\"kind\":\"dev\",\"name\":\"trybuild\",\"req\":\"^1.0.108\"},{\"name\":\"zmij\",\"req\":\"^1.0\"}],\"features\":{\"alloc\":[\"serde_core/alloc\"],\"arbitrary_precision\":[],\"default\":[\"std\"],\"float_roundtrip\":[],\"preserve_order\":[\"indexmap\",\"std\"],\"raw_value\":[],\"std\":[\"memchr/std\",\"serde_core/std\"],\"unbounded_depth\":[]}}", "serde_json_1.0.150": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"automod\",\"req\":\"^1.0.11\"},{\"name\":\"indexmap\",\"optional\":true,\"req\":\"^2.2.3\"},{\"kind\":\"dev\",\"name\":\"indoc\",\"req\":\"^2.0.2\"},{\"name\":\"itoa\",\"req\":\"^1.0\"},{\"default_features\":false,\"name\":\"memchr\",\"req\":\"^2\"},{\"kind\":\"dev\",\"name\":\"ref-cast\",\"req\":\"^1.0.18\"},{\"kind\":\"dev\",\"name\":\"rustversion\",\"req\":\"^1.0.13\"},{\"default_features\":false,\"name\":\"serde\",\"req\":\"^1.0.220\",\"target\":\"cfg(any())\"},{\"features\":[\"derive\"],\"kind\":\"dev\",\"name\":\"serde\",\"req\":\"^1.0.194\"},{\"kind\":\"dev\",\"name\":\"serde_bytes\",\"req\":\"^0.11.10\"},{\"default_features\":false,\"name\":\"serde_core\",\"req\":\"^1.0.220\"},{\"kind\":\"dev\",\"name\":\"serde_derive\",\"req\":\"^1.0.166\"},{\"kind\":\"dev\",\"name\":\"serde_stacker\",\"req\":\"^0.1.8\"},{\"features\":[\"diff\"],\"kind\":\"dev\",\"name\":\"trybuild\",\"req\":\"^1.0.108\"},{\"name\":\"zmij\",\"req\":\"^1.0\"}],\"features\":{\"alloc\":[\"serde_core/alloc\"],\"arbitrary_precision\":[],\"default\":[\"std\"],\"float_roundtrip\":[],\"preserve_order\":[\"indexmap\",\"std\"],\"raw_value\":[],\"std\":[\"memchr/std\",\"serde_core/std\"],\"unbounded_depth\":[]}}", "serde_test_1.0.177": "{\"dependencies\":[{\"name\":\"serde\",\"req\":\"^1.0.69\"},{\"features\":[\"rc\"],\"kind\":\"dev\",\"name\":\"serde\",\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"serde_derive\",\"req\":\"^1\"}],\"features\":{}}", "serde_urlencoded_0.7.1": "{\"dependencies\":[{\"name\":\"form_urlencoded\",\"req\":\"^1\"},{\"name\":\"itoa\",\"req\":\"^1\"},{\"name\":\"ryu\",\"req\":\"^1\"},{\"name\":\"serde\",\"req\":\"^1.0.69\"},{\"kind\":\"dev\",\"name\":\"serde_derive\",\"req\":\"^1\"}],\"features\":{}}", - "serde_with_3.15.1": "{\"dependencies\":[{\"default_features\":false,\"name\":\"base64\",\"optional\":true,\"req\":\"^0.22.1\"},{\"default_features\":false,\"features\":[\"serde\"],\"name\":\"chrono_0_4\",\"optional\":true,\"package\":\"chrono\",\"req\":\"^0.4.20\"},{\"name\":\"document-features\",\"optional\":true,\"req\":\"^0.2.7\"},{\"kind\":\"dev\",\"name\":\"expect-test\",\"req\":\"^1.5.1\"},{\"kind\":\"dev\",\"name\":\"fnv\",\"req\":\"^1.0.6\"},{\"kind\":\"dev\",\"name\":\"glob\",\"req\":\"^0.3.3\"},{\"default_features\":false,\"features\":[\"serde\"],\"name\":\"hashbrown_0_14\",\"optional\":true,\"package\":\"hashbrown\",\"req\":\"^0.14.0\"},{\"default_features\":false,\"features\":[\"serde\"],\"name\":\"hashbrown_0_15\",\"optional\":true,\"package\":\"hashbrown\",\"req\":\"^0.15.0\"},{\"default_features\":false,\"features\":[\"serde\"],\"name\":\"hashbrown_0_16\",\"optional\":true,\"package\":\"hashbrown\",\"req\":\"^0.16.0\"},{\"default_features\":false,\"name\":\"hex\",\"optional\":true,\"req\":\"^0.4.3\"},{\"default_features\":false,\"features\":[\"serde-1\"],\"name\":\"indexmap_1\",\"optional\":true,\"package\":\"indexmap\",\"req\":\"^1.8\"},{\"default_features\":false,\"features\":[\"serde\"],\"name\":\"indexmap_2\",\"optional\":true,\"package\":\"indexmap\",\"req\":\"^2.0\"},{\"default_features\":false,\"features\":[\"resolve-file\"],\"kind\":\"dev\",\"name\":\"jsonschema\",\"req\":\"^0.33.0\"},{\"kind\":\"dev\",\"name\":\"mime\",\"req\":\"^0.3.16\"},{\"kind\":\"dev\",\"name\":\"pretty_assertions\",\"req\":\"^1.4.0\"},{\"default_features\":false,\"features\":[\"std\"],\"kind\":\"dev\",\"name\":\"regex\",\"req\":\"^1.12.1\"},{\"kind\":\"dev\",\"name\":\"rmp-serde\",\"req\":\"^1.3.0\"},{\"kind\":\"dev\",\"name\":\"ron\",\"req\":\"^0.11\"},{\"kind\":\"dev\",\"name\":\"rustversion\",\"req\":\"^1.0.22\"},{\"default_features\":false,\"name\":\"schemars_0_8\",\"optional\":true,\"package\":\"schemars\",\"req\":\"^0.8.16\"},{\"kind\":\"dev\",\"name\":\"schemars_0_8\",\"package\":\"schemars\",\"req\":\"^0.8.16\"},{\"default_features\":false,\"name\":\"schemars_0_9\",\"optional\":true,\"package\":\"schemars\",\"req\":\"^0.9.0\"},{\"kind\":\"dev\",\"name\":\"schemars_0_9\",\"package\":\"schemars\",\"req\":\"^0.9.0\"},{\"default_features\":false,\"name\":\"schemars_1\",\"optional\":true,\"package\":\"schemars\",\"req\":\"^1.0.2\"},{\"kind\":\"dev\",\"name\":\"schemars_1\",\"package\":\"schemars\",\"req\":\"^1.0.2\"},{\"default_features\":false,\"features\":[\"derive\"],\"kind\":\"dev\",\"name\":\"serde\",\"req\":\"^1.0.152\"},{\"kind\":\"dev\",\"name\":\"serde-xml-rs\",\"req\":\"^0.8.1\"},{\"default_features\":false,\"features\":[\"result\"],\"name\":\"serde_core\",\"req\":\"^1.0.225\"},{\"default_features\":false,\"name\":\"serde_json\",\"optional\":true,\"req\":\"^1.0.145\"},{\"features\":[\"preserve_order\"],\"kind\":\"dev\",\"name\":\"serde_json\",\"req\":\"^1.0.25\"},{\"kind\":\"dev\",\"name\":\"serde_test\",\"req\":\"^1.0.124\"},{\"name\":\"serde_with_macros\",\"optional\":true,\"req\":\"=3.15.1\"},{\"kind\":\"dev\",\"name\":\"serde_yaml\",\"req\":\"^0.9.2\"},{\"default_features\":false,\"name\":\"time_0_3\",\"optional\":true,\"package\":\"time\",\"req\":\"~0.3.36\"}],\"features\":{\"alloc\":[\"serde_core/alloc\",\"base64?/alloc\",\"chrono_0_4?/alloc\",\"hex?/alloc\",\"serde_json?/alloc\",\"time_0_3?/alloc\"],\"base64\":[\"dep:base64\",\"alloc\"],\"chrono\":[\"chrono_0_4\"],\"chrono_0_4\":[\"dep:chrono_0_4\"],\"default\":[\"std\",\"macros\"],\"guide\":[\"dep:document-features\",\"macros\",\"std\"],\"hashbrown_0_14\":[\"dep:hashbrown_0_14\",\"alloc\"],\"hashbrown_0_15\":[\"dep:hashbrown_0_15\",\"alloc\"],\"hashbrown_0_16\":[\"dep:hashbrown_0_16\",\"alloc\"],\"hex\":[\"dep:hex\",\"alloc\"],\"indexmap\":[\"indexmap_1\"],\"indexmap_1\":[\"dep:indexmap_1\",\"alloc\"],\"indexmap_2\":[\"dep:indexmap_2\",\"alloc\"],\"json\":[\"dep:serde_json\",\"alloc\"],\"macros\":[\"dep:serde_with_macros\"],\"schemars_0_8\":[\"dep:schemars_0_8\",\"std\",\"serde_with_macros?/schemars_0_8\"],\"schemars_0_9\":[\"dep:schemars_0_9\",\"alloc\",\"serde_with_macros?/schemars_0_9\",\"dep:serde_json\"],\"schemars_1\":[\"dep:schemars_1\",\"alloc\",\"serde_with_macros?/schemars_1\",\"dep:serde_json\"],\"std\":[\"alloc\",\"serde_core/std\",\"chrono_0_4?/clock\",\"chrono_0_4?/std\",\"indexmap_1?/std\",\"indexmap_2?/std\",\"time_0_3?/serde-well-known\",\"time_0_3?/std\",\"schemars_0_9?/std\",\"schemars_1?/std\"],\"time_0_3\":[\"dep:time_0_3\"]}}", - "serde_with_macros_3.15.1": "{\"dependencies\":[{\"name\":\"darling\",\"req\":\"^0.21.0\"},{\"kind\":\"dev\",\"name\":\"expect-test\",\"req\":\"^1.5.1\"},{\"kind\":\"dev\",\"name\":\"glob\",\"req\":\"^0.3.3\"},{\"kind\":\"dev\",\"name\":\"pretty_assertions\",\"req\":\"^1.4.0\"},{\"name\":\"proc-macro2\",\"req\":\"^1.0.1\"},{\"name\":\"quote\",\"req\":\"^1.0.0\"},{\"default_features\":false,\"features\":[\"std\"],\"kind\":\"dev\",\"name\":\"regex\",\"req\":\"^1.12.1\"},{\"kind\":\"dev\",\"name\":\"rustversion\",\"req\":\"^1.0.22\"},{\"features\":[\"derive\"],\"kind\":\"dev\",\"name\":\"serde\",\"req\":\"^1.0.152\"},{\"kind\":\"dev\",\"name\":\"serde_json\",\"req\":\"^1.0.25\"},{\"features\":[\"extra-traits\",\"full\",\"parsing\"],\"name\":\"syn\",\"req\":\"^2.0.0\"},{\"kind\":\"dev\",\"name\":\"trybuild\",\"req\":\"^1.0.111\"}],\"features\":{\"schemars_0_8\":[],\"schemars_0_9\":[],\"schemars_1\":[]}}", + "serde_with_3.21.0": "{\"dependencies\":[{\"default_features\":false,\"name\":\"base64\",\"optional\":true,\"req\":\"^0.22.1\"},{\"default_features\":false,\"name\":\"bs58\",\"optional\":true,\"req\":\"^0.5.1\"},{\"default_features\":false,\"features\":[\"serde\"],\"name\":\"chrono_0_4\",\"optional\":true,\"package\":\"chrono\",\"req\":\"^0.4.20\"},{\"name\":\"document-features\",\"optional\":true,\"req\":\"^0.2.7\"},{\"kind\":\"dev\",\"name\":\"expect-test\",\"req\":\"^1.5.1\"},{\"kind\":\"dev\",\"name\":\"fnv\",\"req\":\"^1.0.6\"},{\"kind\":\"dev\",\"name\":\"glob\",\"req\":\"^0.3.3\"},{\"default_features\":false,\"features\":[\"serde\"],\"name\":\"hashbrown_0_14\",\"optional\":true,\"package\":\"hashbrown\",\"req\":\"^0.14.0\"},{\"default_features\":false,\"features\":[\"serde\"],\"name\":\"hashbrown_0_15\",\"optional\":true,\"package\":\"hashbrown\",\"req\":\"^0.15.0\"},{\"default_features\":false,\"features\":[\"serde\"],\"name\":\"hashbrown_0_16\",\"optional\":true,\"package\":\"hashbrown\",\"req\":\"^0.16.0\"},{\"default_features\":false,\"features\":[\"serde\"],\"name\":\"hashbrown_0_17\",\"optional\":true,\"package\":\"hashbrown\",\"req\":\"^0.17.0\"},{\"default_features\":false,\"name\":\"hex\",\"optional\":true,\"req\":\"^0.4.3\"},{\"default_features\":false,\"features\":[\"serde-1\"],\"name\":\"indexmap_1\",\"optional\":true,\"package\":\"indexmap\",\"req\":\"^1.8\"},{\"default_features\":false,\"features\":[\"serde\"],\"name\":\"indexmap_2\",\"optional\":true,\"package\":\"indexmap\",\"req\":\"^2.0\"},{\"default_features\":false,\"features\":[\"resolve-file\"],\"kind\":\"dev\",\"name\":\"jsonschema\",\"req\":\"^0.33.0\"},{\"kind\":\"dev\",\"name\":\"mime\",\"req\":\"^0.3.16\"},{\"kind\":\"dev\",\"name\":\"pretty_assertions\",\"req\":\"^1.4.0\"},{\"default_features\":false,\"features\":[\"std\"],\"kind\":\"dev\",\"name\":\"regex\",\"req\":\"^1.12.1\"},{\"kind\":\"dev\",\"name\":\"rmp-serde\",\"req\":\"^1.3.0\"},{\"kind\":\"dev\",\"name\":\"ron\",\"req\":\"^0.12\"},{\"kind\":\"dev\",\"name\":\"rustversion\",\"req\":\"^1.0.22\"},{\"default_features\":false,\"name\":\"schemars_0_8\",\"optional\":true,\"package\":\"schemars\",\"req\":\"^0.8.16\"},{\"kind\":\"dev\",\"name\":\"schemars_0_8\",\"package\":\"schemars\",\"req\":\"^0.8.16\"},{\"default_features\":false,\"name\":\"schemars_0_9\",\"optional\":true,\"package\":\"schemars\",\"req\":\"^0.9.0\"},{\"kind\":\"dev\",\"name\":\"schemars_0_9\",\"package\":\"schemars\",\"req\":\"^0.9.0\"},{\"default_features\":false,\"name\":\"schemars_1\",\"optional\":true,\"package\":\"schemars\",\"req\":\"^1.0.2\"},{\"kind\":\"dev\",\"name\":\"schemars_1\",\"package\":\"schemars\",\"req\":\"^1.0.2\"},{\"default_features\":false,\"features\":[\"derive\"],\"kind\":\"dev\",\"name\":\"serde\",\"req\":\"^1.0.152\"},{\"kind\":\"dev\",\"name\":\"serde-xml-rs\",\"req\":\"^0.8.1\"},{\"default_features\":false,\"features\":[\"result\"],\"name\":\"serde_core\",\"req\":\"^1.0.225\"},{\"default_features\":false,\"name\":\"serde_json\",\"optional\":true,\"req\":\"^1.0.145\"},{\"features\":[\"preserve_order\"],\"kind\":\"dev\",\"name\":\"serde_json\",\"req\":\"^1.0.25\"},{\"kind\":\"dev\",\"name\":\"serde_test\",\"req\":\"^1.0.124\"},{\"name\":\"serde_with_macros\",\"optional\":true,\"req\":\"=3.21.0\"},{\"default_features\":false,\"name\":\"smallvec_1\",\"optional\":true,\"package\":\"smallvec\",\"req\":\"^1\"},{\"default_features\":false,\"name\":\"time_0_3\",\"optional\":true,\"package\":\"time\",\"req\":\"~0.3.47\"},{\"kind\":\"dev\",\"name\":\"yaml_serde\",\"req\":\"^0.10.3\"}],\"features\":{\"alloc\":[\"serde_core/alloc\",\"base64?/alloc\",\"bs58?/alloc\",\"chrono_0_4?/alloc\",\"hex?/alloc\",\"serde_json?/alloc\",\"time_0_3?/alloc\"],\"base58\":[\"dep:bs58\",\"alloc\"],\"base64\":[\"dep:base64\",\"alloc\"],\"chrono\":[\"chrono_0_4\"],\"chrono_0_4\":[\"dep:chrono_0_4\"],\"default\":[\"std\",\"macros\"],\"guide\":[\"dep:document-features\",\"macros\",\"std\"],\"hashbrown_0_14\":[\"dep:hashbrown_0_14\",\"alloc\"],\"hashbrown_0_15\":[\"dep:hashbrown_0_15\",\"alloc\"],\"hashbrown_0_16\":[\"dep:hashbrown_0_16\",\"alloc\"],\"hashbrown_0_17\":[\"dep:hashbrown_0_17\",\"alloc\"],\"hex\":[\"dep:hex\",\"alloc\"],\"indexmap\":[\"indexmap_1\"],\"indexmap_1\":[\"dep:indexmap_1\",\"alloc\"],\"indexmap_2\":[\"dep:indexmap_2\",\"alloc\"],\"json\":[\"dep:serde_json\",\"alloc\"],\"macros\":[\"dep:serde_with_macros\"],\"schemars_0_8\":[\"dep:schemars_0_8\",\"std\",\"serde_with_macros?/schemars_0_8\",\"dep:serde_json\"],\"schemars_0_9\":[\"dep:schemars_0_9\",\"alloc\",\"serde_with_macros?/schemars_0_9\",\"dep:serde_json\"],\"schemars_1\":[\"dep:schemars_1\",\"alloc\",\"serde_with_macros?/schemars_1\",\"dep:serde_json\"],\"smallvec_1\":[\"dep:smallvec_1\"],\"std\":[\"alloc\",\"bs58?/std\",\"serde_core/std\",\"chrono_0_4?/clock\",\"chrono_0_4?/std\",\"indexmap_1?/std\",\"indexmap_2?/std\",\"time_0_3?/serde-well-known\",\"time_0_3?/std\",\"schemars_0_9?/std\",\"schemars_1?/std\"],\"time_0_3\":[\"dep:time_0_3\"]}}", + "serde_with_macros_3.21.0": "{\"dependencies\":[{\"name\":\"darling\",\"req\":\"^0.23.0\"},{\"kind\":\"dev\",\"name\":\"expect-test\",\"req\":\"^1.5.1\"},{\"kind\":\"dev\",\"name\":\"glob\",\"req\":\"^0.3.3\"},{\"kind\":\"dev\",\"name\":\"pretty_assertions\",\"req\":\"^1.4.0\"},{\"name\":\"proc-macro2\",\"req\":\"^1.0.1\"},{\"name\":\"quote\",\"req\":\"^1.0.0\"},{\"default_features\":false,\"features\":[\"std\"],\"kind\":\"dev\",\"name\":\"regex\",\"req\":\"^1.12.1\"},{\"kind\":\"dev\",\"name\":\"rustversion\",\"req\":\"^1.0.22\"},{\"features\":[\"derive\"],\"kind\":\"dev\",\"name\":\"serde\",\"req\":\"^1.0.152\"},{\"kind\":\"dev\",\"name\":\"serde_json\",\"req\":\"^1.0.25\"},{\"features\":[\"extra-traits\",\"full\",\"parsing\"],\"name\":\"syn\",\"req\":\"^2.0.0\"},{\"kind\":\"dev\",\"name\":\"trybuild\",\"req\":\"^1.0.111\"}],\"features\":{\"schemars_0_8\":[],\"schemars_0_9\":[],\"schemars_1\":[]}}", "serial_test_3.5.0": "{\"dependencies\":[{\"name\":\"document-features\",\"optional\":true,\"req\":\"^0.2\"},{\"default_features\":false,\"name\":\"env_logger\",\"optional\":true,\"req\":\">=0.6.1\"},{\"default_features\":false,\"features\":[\"std\"],\"name\":\"fslock\",\"optional\":true,\"req\":\"^0.2\"},{\"default_features\":false,\"features\":[\"std\"],\"name\":\"futures-executor\",\"optional\":true,\"req\":\"^0.3\"},{\"default_features\":false,\"features\":[\"std\"],\"name\":\"futures-util\",\"optional\":true,\"req\":\"^0.3\"},{\"default_features\":false,\"features\":[\"use_std\"],\"kind\":\"dev\",\"name\":\"itertools\",\"req\":\">=0.4\"},{\"name\":\"log\",\"optional\":true,\"req\":\">=0.4.4\"},{\"default_features\":false,\"features\":[\"std\"],\"name\":\"once_cell\",\"req\":\"^1.19\"},{\"default_features\":false,\"name\":\"parking_lot\",\"req\":\"^0.12\"},{\"name\":\"serial_test_derive\",\"req\":\"~3.5.0\"}],\"features\":{\"async\":[\"dep:futures-executor\",\"dep:futures-util\",\"serial_test_derive/async\"],\"default\":[\"logging\",\"async\"],\"docsrs\":[\"dep:document-features\"],\"file_locks\":[\"dep:fslock\"],\"logging\":[\"dep:log\"],\"test_logging\":[\"logging\",\"dep:env_logger\",\"serial_test_derive/test_logging\"]}}", "serial_test_derive_3.5.0": "{\"dependencies\":[{\"default_features\":false,\"kind\":\"dev\",\"name\":\"env_logger\",\"req\":\">=0.6.1\"},{\"default_features\":false,\"kind\":\"dev\",\"name\":\"prettyplease\",\"req\":\"^0.2\"},{\"default_features\":false,\"features\":[\"proc-macro\"],\"name\":\"proc-macro2\",\"req\":\"^1.0.60\"},{\"default_features\":false,\"name\":\"quote\",\"req\":\"^1\"},{\"default_features\":false,\"features\":[\"full\",\"printing\",\"parsing\",\"clone-impls\"],\"name\":\"syn\",\"req\":\"^2\"}],\"features\":{\"async\":[],\"default\":[],\"file_locks\":[],\"test_logging\":[]}}", "sha1_0.10.6": "{\"dependencies\":[{\"name\":\"cfg-if\",\"req\":\"^1.0\"},{\"name\":\"cpufeatures\",\"req\":\"^0.2\",\"target\":\"cfg(any(target_arch = \\\"aarch64\\\", target_arch = \\\"x86\\\", target_arch = \\\"x86_64\\\"))\"},{\"name\":\"digest\",\"req\":\"^0.10.7\"},{\"features\":[\"dev\"],\"kind\":\"dev\",\"name\":\"digest\",\"req\":\"^0.10.7\"},{\"kind\":\"dev\",\"name\":\"hex-literal\",\"req\":\"^0.2.2\"},{\"name\":\"sha1-asm\",\"optional\":true,\"req\":\"^0.5\",\"target\":\"cfg(any(target_arch = \\\"aarch64\\\", target_arch = \\\"x86\\\", target_arch = \\\"x86_64\\\"))\"}],\"features\":{\"asm\":[\"sha1-asm\"],\"compress\":[],\"default\":[\"std\"],\"force-soft\":[],\"loongarch64_asm\":[],\"oid\":[\"digest/oid\"],\"std\":[\"digest/std\"]}}", @@ -1225,10 +1177,12 @@ "sha2_0.10.9": "{\"dependencies\":[{\"name\":\"cfg-if\",\"req\":\"^1.0\"},{\"name\":\"cpufeatures\",\"req\":\"^0.2\",\"target\":\"cfg(any(target_arch = \\\"aarch64\\\", target_arch = \\\"x86_64\\\", target_arch = \\\"x86\\\"))\"},{\"name\":\"digest\",\"req\":\"^0.10.7\"},{\"features\":[\"dev\"],\"kind\":\"dev\",\"name\":\"digest\",\"req\":\"^0.10.7\"},{\"kind\":\"dev\",\"name\":\"hex-literal\",\"req\":\"^0.2.2\"},{\"name\":\"sha2-asm\",\"optional\":true,\"req\":\"^0.6.1\",\"target\":\"cfg(any(target_arch = \\\"aarch64\\\", target_arch = \\\"x86_64\\\", target_arch = \\\"x86\\\"))\"}],\"features\":{\"asm\":[\"sha2-asm\"],\"asm-aarch64\":[\"asm\"],\"compress\":[],\"default\":[\"std\"],\"force-soft\":[],\"force-soft-compact\":[],\"loongarch64_asm\":[],\"oid\":[\"digest/oid\"],\"std\":[\"digest/std\"]}}", "sharded-slab_0.1.7": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"criterion\",\"req\":\"^0.3\"},{\"kind\":\"dev\",\"name\":\"indexmap\",\"req\":\"^1\"},{\"name\":\"lazy_static\",\"req\":\"^1\"},{\"features\":[\"checkpoint\"],\"name\":\"loom\",\"optional\":true,\"req\":\"^0.5\",\"target\":\"cfg(loom)\"},{\"features\":[\"checkpoint\"],\"kind\":\"dev\",\"name\":\"loom\",\"req\":\"^0.5\",\"target\":\"cfg(loom)\"},{\"kind\":\"dev\",\"name\":\"memory-stats\",\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"proptest\",\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"slab\",\"req\":\"^0.4.2\"}],\"features\":{}}", "shellexpand_3.1.1": "{\"dependencies\":[{\"name\":\"bstr\",\"optional\":true,\"req\":\"^1.0.0-pre.2\"},{\"name\":\"dirs\",\"optional\":true,\"req\":\">=4, <7\"},{\"name\":\"os_str_bytes\",\"optional\":true,\"req\":\">=5, <7\"}],\"features\":{\"base-0\":[],\"default\":[\"base-0\",\"tilde\"],\"full\":[\"full-msrv-1-51\"],\"full-msrv-1-31\":[\"base-0\",\"tilde\"],\"full-msrv-1-51\":[\"full-msrv-1-31\",\"path\"],\"path\":[\"bstr\",\"os_str_bytes\"],\"tilde\":[\"dirs\"]}}", + "shlex_1.3.0": "{\"dependencies\":[],\"features\":{\"default\":[\"std\"],\"std\":[]}}", "shlex_2.0.1": "{\"dependencies\":[],\"features\":{\"default\":[\"std\"],\"std\":[]}}", "signal-hook-registry_1.4.8": "{\"dependencies\":[{\"name\":\"errno\",\"req\":\">=0.2, <0.4\"},{\"name\":\"libc\",\"req\":\"^0.2\"},{\"kind\":\"dev\",\"name\":\"signal-hook\",\"req\":\"~0.3\"}],\"features\":{}}", "signature_2.2.0": "{\"dependencies\":[{\"name\":\"derive\",\"optional\":true,\"package\":\"signature_derive\",\"req\":\"^2\"},{\"default_features\":false,\"name\":\"digest\",\"optional\":true,\"req\":\"^0.10.6\"},{\"kind\":\"dev\",\"name\":\"hex-literal\",\"req\":\"^0.4\"},{\"default_features\":false,\"name\":\"rand_core\",\"optional\":true,\"req\":\"^0.6.4\"},{\"default_features\":false,\"kind\":\"dev\",\"name\":\"sha2\",\"req\":\"^0.10\"}],\"features\":{\"alloc\":[],\"std\":[\"alloc\",\"rand_core?/std\"]}}", "simd-adler32_0.3.7": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"adler\",\"req\":\"^1.0.2\"},{\"kind\":\"dev\",\"name\":\"adler32\",\"req\":\"^1.2.0\"},{\"kind\":\"dev\",\"name\":\"criterion\",\"req\":\"^0.3\"},{\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.8\"}],\"features\":{\"const-generics\":[],\"default\":[\"std\",\"const-generics\"],\"nightly\":[],\"std\":[]}}", + "simd-adler32_0.3.9": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"adler\",\"req\":\"^1.0.2\"},{\"kind\":\"dev\",\"name\":\"adler32\",\"req\":\"^1.2.0\"},{\"kind\":\"dev\",\"name\":\"criterion\",\"req\":\"^0.3\"},{\"features\":[\"small_rng\"],\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.8\"}],\"features\":{\"const-generics\":[],\"default\":[\"std\",\"const-generics\"],\"nightly\":[],\"std\":[]}}", "simd_cesu8_1.1.1": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"cesu8\",\"req\":\"^1.1.0\"},{\"features\":[\"html_reports\"],\"kind\":\"dev\",\"name\":\"criterion\",\"req\":\"^0.5.1\"},{\"kind\":\"build\",\"name\":\"rustc_version\",\"req\":\"^0.4.0\"},{\"default_features\":false,\"name\":\"simdutf8\",\"req\":\"^0.1.4\"}],\"features\":{\"bench\":[],\"default\":[\"std\"],\"nightly\":[],\"std\":[\"simdutf8/std\"]}}", "simdutf8_0.1.5": "{\"dependencies\":[],\"features\":{\"aarch64_neon\":[],\"aarch64_neon_prefetch\":[],\"default\":[\"std\"],\"hints\":[],\"public_imp\":[],\"std\":[]}}", "simple_asn1_0.6.3": "{\"dependencies\":[{\"default_features\":false,\"name\":\"num-bigint\",\"req\":\"^0.4\"},{\"default_features\":false,\"name\":\"num-traits\",\"req\":\"^0.2\"},{\"kind\":\"dev\",\"name\":\"quickcheck\",\"req\":\"^1.0.3\"},{\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.8.4\"},{\"default_features\":false,\"name\":\"thiserror\",\"req\":\"^2\"},{\"default_features\":false,\"features\":[\"formatting\",\"macros\",\"parsing\"],\"name\":\"time\",\"req\":\"^0.3\"},{\"default_features\":false,\"features\":[\"formatting\",\"macros\",\"parsing\",\"quickcheck\"],\"kind\":\"dev\",\"name\":\"time\",\"req\":\"^0.3\"}],\"features\":{}}", @@ -1246,6 +1200,7 @@ "strsim_0.11.1": "{\"dependencies\":[],\"features\":{}}", "subtle_2.6.1": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.8\"}],\"features\":{\"const-generics\":[],\"core_hint_black_box\":[],\"default\":[\"std\",\"i128\"],\"i128\":[],\"nightly\":[],\"std\":[]}}", "syn_2.0.117": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"anyhow\",\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"automod\",\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"flate2\",\"req\":\"^1\",\"target\":\"cfg(not(miri))\"},{\"kind\":\"dev\",\"name\":\"insta\",\"req\":\"^1\"},{\"default_features\":false,\"name\":\"proc-macro2\",\"req\":\"^1.0.91\"},{\"default_features\":false,\"name\":\"quote\",\"optional\":true,\"req\":\"^1.0.35\"},{\"kind\":\"dev\",\"name\":\"rayon\",\"req\":\"^1\",\"target\":\"cfg(not(miri))\"},{\"kind\":\"dev\",\"name\":\"ref-cast\",\"req\":\"^1\"},{\"features\":[\"blocking\"],\"kind\":\"dev\",\"name\":\"reqwest\",\"req\":\"^0.13\",\"target\":\"cfg(not(miri))\"},{\"kind\":\"dev\",\"name\":\"rustversion\",\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"syn-test-suite\",\"req\":\"^0\"},{\"kind\":\"dev\",\"name\":\"tar\",\"req\":\"^0.4.16\",\"target\":\"cfg(not(miri))\"},{\"kind\":\"dev\",\"name\":\"termcolor\",\"req\":\"^1\"},{\"name\":\"unicode-ident\",\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"walkdir\",\"req\":\"^2.3.2\",\"target\":\"cfg(not(miri))\"}],\"features\":{\"clone-impls\":[],\"default\":[\"derive\",\"parsing\",\"printing\",\"clone-impls\",\"proc-macro\"],\"derive\":[],\"extra-traits\":[],\"fold\":[],\"full\":[],\"parsing\":[],\"printing\":[\"dep:quote\"],\"proc-macro\":[\"proc-macro2/proc-macro\",\"quote?/proc-macro\"],\"test\":[\"syn-test-suite/all-features\"],\"visit\":[],\"visit-mut\":[]}}", + "syn_3.0.1": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"anyhow\",\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"automod\",\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"flate2\",\"req\":\"^1\",\"target\":\"cfg(not(miri))\"},{\"kind\":\"dev\",\"name\":\"insta\",\"req\":\"^1\"},{\"default_features\":false,\"name\":\"proc-macro2\",\"req\":\"^1.0.91\"},{\"default_features\":false,\"name\":\"quote\",\"optional\":true,\"req\":\"^1.0.35\"},{\"kind\":\"dev\",\"name\":\"rayon\",\"req\":\"^1\",\"target\":\"cfg(not(miri))\"},{\"kind\":\"dev\",\"name\":\"ref-cast\",\"req\":\"^1\"},{\"features\":[\"blocking\"],\"kind\":\"dev\",\"name\":\"reqwest\",\"req\":\"^0.13\",\"target\":\"cfg(not(miri))\"},{\"kind\":\"dev\",\"name\":\"rustversion\",\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"syn-test-suite\",\"req\":\"^0\"},{\"kind\":\"dev\",\"name\":\"tar\",\"req\":\"^0.4.16\",\"target\":\"cfg(not(miri))\"},{\"kind\":\"dev\",\"name\":\"termcolor\",\"req\":\"^1\"},{\"name\":\"unicode-ident\",\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"walkdir\",\"req\":\"^2.3.2\",\"target\":\"cfg(not(miri))\"}],\"features\":{\"clone-impls\":[],\"default\":[\"derive\",\"parsing\",\"printing\",\"clone-impls\",\"proc-macro\"],\"derive\":[],\"extra-traits\":[],\"fold\":[],\"full\":[],\"parsing\":[],\"printing\":[\"dep:quote\"],\"proc-macro\":[\"proc-macro2/proc-macro\",\"quote?/proc-macro\"],\"test\":[\"syn-test-suite/all-features\"],\"visit\":[],\"visit-mut\":[]}}", "sync_wrapper_1.0.2": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"futures\",\"req\":\"^0.3\"},{\"default_features\":false,\"name\":\"futures-core\",\"optional\":true,\"req\":\"^0.3\"},{\"kind\":\"dev\",\"name\":\"pin-project-lite\",\"req\":\"^0.2.7\"}],\"features\":{\"futures\":[\"futures-core\"]}}", "synstructure_0.13.2": "{\"dependencies\":[{\"default_features\":false,\"name\":\"proc-macro2\",\"req\":\"^1.0.60\"},{\"default_features\":false,\"name\":\"quote\",\"req\":\"^1\"},{\"default_features\":false,\"features\":[\"derive\",\"parsing\",\"printing\",\"clone-impls\",\"visit\",\"extra-traits\"],\"name\":\"syn\",\"req\":\"^2\"},{\"kind\":\"dev\",\"name\":\"synstructure_test_traits\",\"req\":\"^0.1\"}],\"features\":{\"default\":[\"proc-macro\"],\"proc-macro\":[\"proc-macro2/proc-macro\",\"syn/proc-macro\",\"quote/proc-macro\"]}}", "system-configuration-sys_0.6.0": "{\"dependencies\":[{\"name\":\"core-foundation-sys\",\"req\":\"^0.8\"},{\"name\":\"libc\",\"req\":\"^0.2.149\"}],\"features\":{}}", @@ -1262,10 +1217,12 @@ "thiserror_1.0.69": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"anyhow\",\"req\":\"^1.0.73\"},{\"kind\":\"dev\",\"name\":\"ref-cast\",\"req\":\"^1.0.18\"},{\"kind\":\"dev\",\"name\":\"rustversion\",\"req\":\"^1.0.13\"},{\"name\":\"thiserror-impl\",\"req\":\"=1.0.69\"},{\"features\":[\"diff\"],\"kind\":\"dev\",\"name\":\"trybuild\",\"req\":\"^1.0.81\"}],\"features\":{}}", "thiserror_2.0.18": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"anyhow\",\"req\":\"^1.0.73\"},{\"kind\":\"dev\",\"name\":\"ref-cast\",\"req\":\"^1.0.18\"},{\"kind\":\"dev\",\"name\":\"rustversion\",\"req\":\"^1.0.13\"},{\"name\":\"thiserror-impl\",\"req\":\"=2.0.18\"},{\"features\":[\"diff\"],\"kind\":\"dev\",\"name\":\"trybuild\",\"req\":\"^1.0.108\"}],\"features\":{\"default\":[\"std\"],\"std\":[]}}", "thread_local_1.1.9": "{\"dependencies\":[{\"name\":\"cfg-if\",\"req\":\"^1.0.0\"},{\"kind\":\"dev\",\"name\":\"criterion\",\"req\":\"^0.5.1\"}],\"features\":{\"nightly\":[]}}", + "threadpool_1.8.1": "{\"dependencies\":[{\"name\":\"num_cpus\",\"req\":\"^1.13\"}],\"features\":{}}", "time-core_0.1.8": "{\"dependencies\":[],\"features\":{\"large-dates\":[]}}", "time-macros_0.2.27": "{\"dependencies\":[{\"name\":\"num-conv\",\"req\":\"^0.2.0\"},{\"name\":\"time-core\",\"req\":\"=0.1.8\"}],\"features\":{\"formatting\":[],\"large-dates\":[],\"parsing\":[],\"serde\":[]}}", "time_0.3.47": "{\"dependencies\":[{\"default_features\":false,\"kind\":\"dev\",\"name\":\"criterion\",\"req\":\"^0.8.1\",\"target\":\"cfg(bench)\"},{\"features\":[\"powerfmt\"],\"name\":\"deranged\",\"req\":\"^0.5.2\"},{\"name\":\"itoa\",\"optional\":true,\"req\":\"^1.0.1\"},{\"name\":\"js-sys\",\"optional\":true,\"req\":\"^0.3.58\",\"target\":\"cfg(all(target_family = \\\"wasm\\\", not(any(target_os = \\\"emscripten\\\", target_os = \\\"wasi\\\"))))\"},{\"name\":\"libc\",\"optional\":true,\"req\":\"^0.2.98\",\"target\":\"cfg(target_family = \\\"unix\\\")\"},{\"name\":\"num-conv\",\"req\":\"^0.2.0\"},{\"kind\":\"dev\",\"name\":\"num-conv\",\"req\":\"^0.2.0\"},{\"name\":\"num_threads\",\"optional\":true,\"req\":\"^0.1.2\",\"target\":\"cfg(target_family = \\\"unix\\\")\"},{\"default_features\":false,\"name\":\"powerfmt\",\"req\":\"^0.2.0\"},{\"default_features\":false,\"name\":\"quickcheck\",\"optional\":true,\"req\":\"^1.0.3\"},{\"kind\":\"dev\",\"name\":\"quickcheck_macros\",\"req\":\"^1.0.0\"},{\"default_features\":false,\"name\":\"rand08\",\"optional\":true,\"package\":\"rand\",\"req\":\"^0.8.4\"},{\"default_features\":false,\"kind\":\"dev\",\"name\":\"rand08\",\"package\":\"rand\",\"req\":\"^0.8.4\"},{\"default_features\":false,\"name\":\"rand09\",\"optional\":true,\"package\":\"rand\",\"req\":\"^0.9.2\"},{\"default_features\":false,\"features\":[\"small_rng\"],\"kind\":\"dev\",\"name\":\"rand09\",\"package\":\"rand\",\"req\":\"^0.9.2\"},{\"default_features\":false,\"kind\":\"dev\",\"name\":\"rstest\",\"req\":\"^0.26.1\"},{\"kind\":\"dev\",\"name\":\"rstest_reuse\",\"req\":\"^0.7.0\"},{\"default_features\":false,\"features\":[\"derive\"],\"kind\":\"dev\",\"name\":\"serde\",\"req\":\"^1.0.184\"},{\"default_features\":false,\"name\":\"serde_core\",\"optional\":true,\"req\":\"^1.0.220\"},{\"kind\":\"dev\",\"name\":\"serde_json\",\"req\":\"^1.0.68\"},{\"kind\":\"dev\",\"name\":\"serde_test\",\"req\":\"^1.0.126\"},{\"name\":\"time-core\",\"req\":\"=0.1.8\"},{\"name\":\"time-macros\",\"optional\":true,\"req\":\"=0.2.27\"},{\"kind\":\"dev\",\"name\":\"time-macros\",\"req\":\"=0.2.27\"},{\"kind\":\"dev\",\"name\":\"trybuild\",\"req\":\"^1.0.102\",\"target\":\"cfg(__ui_tests)\"}],\"features\":{\"alloc\":[\"serde_core?/alloc\"],\"default\":[\"std\"],\"formatting\":[\"dep:itoa\",\"std\",\"time-macros?/formatting\"],\"large-dates\":[\"time-core/large-dates\",\"time-macros?/large-dates\"],\"local-offset\":[\"std\",\"dep:libc\",\"dep:num_threads\"],\"macros\":[\"dep:time-macros\"],\"parsing\":[\"time-macros?/parsing\"],\"quickcheck\":[\"dep:quickcheck\",\"alloc\",\"deranged/quickcheck\"],\"rand\":[\"rand08\",\"rand09\"],\"rand08\":[\"dep:rand08\",\"deranged/rand08\"],\"rand09\":[\"dep:rand09\",\"deranged/rand09\"],\"serde\":[\"dep:serde_core\",\"time-macros?/serde\",\"deranged/serde\"],\"serde-human-readable\":[\"serde\",\"formatting\",\"parsing\"],\"serde-well-known\":[\"serde\",\"formatting\",\"parsing\"],\"std\":[\"alloc\"],\"wasm-bindgen\":[\"dep:js-sys\"]}}", "tiny-keccak_2.0.2": "{\"dependencies\":[{\"name\":\"crunchy\",\"req\":\"^0.2.2\"}],\"features\":{\"cshake\":[],\"default\":[],\"fips202\":[\"keccak\",\"shake\",\"sha3\"],\"k12\":[],\"keccak\":[],\"kmac\":[\"cshake\"],\"parallel_hash\":[\"cshake\"],\"sha3\":[],\"shake\":[],\"sp800\":[\"cshake\",\"kmac\",\"tuple_hash\"],\"tuple_hash\":[\"cshake\"]}}", + "tiny_http_0.12.0": "{\"dependencies\":[{\"name\":\"ascii\",\"req\":\"^1.0\"},{\"name\":\"chunked_transfer\",\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"fdlimit\",\"req\":\"^0.1\"},{\"name\":\"httpdate\",\"req\":\"^1.0.2\"},{\"name\":\"log\",\"req\":\"^0.4.4\"},{\"name\":\"openssl\",\"optional\":true,\"req\":\"^0.10\"},{\"kind\":\"dev\",\"name\":\"rustc-serialize\",\"req\":\"^0.3\"},{\"name\":\"rustls\",\"optional\":true,\"req\":\"^0.20\"},{\"name\":\"rustls-pemfile\",\"optional\":true,\"req\":\"^0.2.1\"},{\"kind\":\"dev\",\"name\":\"sha1\",\"req\":\"^0.6.0\"},{\"name\":\"zeroize\",\"optional\":true,\"req\":\"^1\"}],\"features\":{\"default\":[],\"ssl\":[\"ssl-openssl\"],\"ssl-openssl\":[\"openssl\",\"zeroize\"],\"ssl-rustls\":[\"rustls\",\"rustls-pemfile\",\"zeroize\"]}}", "tinystr_0.8.3": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"bincode\",\"req\":\"^1.3.1\"},{\"kind\":\"dev\",\"name\":\"criterion\",\"req\":\"^0.5.0\",\"target\":\"cfg(not(target_arch = \\\"wasm32\\\"))\"},{\"default_features\":false,\"name\":\"databake\",\"optional\":true,\"req\":\"^0.2.0\"},{\"default_features\":false,\"name\":\"displaydoc\",\"req\":\"^0.2.3\"},{\"default_features\":false,\"features\":[\"use-std\"],\"kind\":\"dev\",\"name\":\"postcard\",\"req\":\"^1.0.3\"},{\"features\":[\"small_rng\"],\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.9\"},{\"default_features\":false,\"name\":\"serde_core\",\"optional\":true,\"req\":\"^1.0.220\"},{\"features\":[\"alloc\"],\"kind\":\"dev\",\"name\":\"serde_json\",\"req\":\"^1.0.45\"},{\"default_features\":false,\"name\":\"zerovec\",\"optional\":true,\"req\":\"^0.11.6\"}],\"features\":{\"alloc\":[\"serde_core?/alloc\",\"zerovec?/alloc\"],\"databake\":[\"dep:databake\"],\"default\":[\"alloc\"],\"serde\":[\"dep:serde_core\"],\"std\":[],\"zerovec\":[\"dep:zerovec\"]}}", "tinyvec_1.11.0": "{\"dependencies\":[{\"name\":\"arbitrary\",\"optional\":true,\"req\":\"^1\"},{\"default_features\":false,\"name\":\"borsh\",\"optional\":true,\"req\":\"^1.2.0\"},{\"kind\":\"dev\",\"name\":\"criterion\",\"req\":\"^0.3.0\"},{\"kind\":\"dev\",\"name\":\"debugger_test\",\"req\":\"^0.1\"},{\"kind\":\"dev\",\"name\":\"debugger_test_parser\",\"req\":\"^0.1\"},{\"name\":\"defmt\",\"optional\":true,\"req\":\"^1.0\"},{\"default_features\":false,\"name\":\"generic-array\",\"optional\":true,\"req\":\"^1.1.1\"},{\"default_features\":false,\"name\":\"serde_core\",\"optional\":true,\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"serde_test\",\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"smallvec\",\"req\":\"^1\"},{\"name\":\"tinyvec_macros\",\"optional\":true,\"req\":\"^0.1\"}],\"features\":{\"alloc\":[\"tinyvec_macros\"],\"debugger_visualizer\":[],\"default\":[],\"experimental_write_impl\":[],\"grab_spare_slice\":[],\"latest_stable_rust\":[\"rustc_1_61\"],\"nightly_slice_partition_dedup\":[],\"real_blackbox\":[\"criterion/real_blackbox\"],\"rustc_1_40\":[],\"rustc_1_55\":[],\"rustc_1_57\":[],\"rustc_1_61\":[\"rustc_1_57\"],\"serde\":[\"dep:serde_core\"],\"std\":[\"alloc\"]}}", "tinyvec_macros_0.1.1": "{\"dependencies\":[],\"features\":{}}", @@ -1302,21 +1259,24 @@ "tracing-test_0.2.5": "{\"dependencies\":[{\"features\":[\"rt-multi-thread\",\"macros\"],\"kind\":\"dev\",\"name\":\"tokio\",\"req\":\"^1\"},{\"default_features\":false,\"features\":[\"std\"],\"kind\":\"dev\",\"name\":\"tracing\",\"req\":\"^0.1\"},{\"name\":\"tracing-core\",\"req\":\"^0.1\"},{\"features\":[\"env-filter\"],\"name\":\"tracing-subscriber\",\"req\":\"^0.3\"},{\"name\":\"tracing-test-macro\",\"req\":\"^0.2.5\"}],\"features\":{\"no-env-filter\":[\"tracing-test-macro/no-env-filter\"]}}", "tracing_0.1.44": "{\"dependencies\":[{\"default_features\":false,\"kind\":\"dev\",\"name\":\"criterion\",\"req\":\"^0.3.6\"},{\"default_features\":false,\"kind\":\"dev\",\"name\":\"futures\",\"req\":\"^0.3.21\"},{\"name\":\"log\",\"optional\":true,\"req\":\"^0.4.17\"},{\"kind\":\"dev\",\"name\":\"log\",\"req\":\"^0.4.17\"},{\"name\":\"pin-project-lite\",\"req\":\"^0.2.9\"},{\"name\":\"tracing-attributes\",\"optional\":true,\"req\":\"^0.1.31\"},{\"default_features\":false,\"name\":\"tracing-core\",\"req\":\"^0.1.36\"},{\"kind\":\"dev\",\"name\":\"wasm-bindgen-test\",\"req\":\"^0.3.38\",\"target\":\"cfg(target_arch = \\\"wasm32\\\")\"}],\"features\":{\"async-await\":[],\"attributes\":[\"tracing-attributes\"],\"default\":[\"std\",\"attributes\"],\"log-always\":[\"log\"],\"max_level_debug\":[],\"max_level_error\":[],\"max_level_info\":[],\"max_level_off\":[],\"max_level_trace\":[],\"max_level_warn\":[],\"release_max_level_debug\":[],\"release_max_level_error\":[],\"release_max_level_info\":[],\"release_max_level_off\":[],\"release_max_level_trace\":[],\"release_max_level_warn\":[],\"std\":[\"tracing-core/std\"],\"valuable\":[\"tracing-core/valuable\"]}}", "try-lock_0.2.5": "{\"dependencies\":[],\"features\":{}}", + "twoway_0.1.8": "{\"dependencies\":[{\"name\":\"galil-seiferas\",\"optional\":true,\"req\":\"^0.1.1\"},{\"kind\":\"dev\",\"name\":\"itertools\",\"req\":\"^0.7.0\"},{\"features\":[\"unstable\"],\"name\":\"jetscii\",\"optional\":true,\"req\":\"^0.3\"},{\"kind\":\"dev\",\"name\":\"macro-attr\",\"req\":\"^0.2\"},{\"default_features\":false,\"name\":\"memchr\",\"req\":\"^2.0\"},{\"kind\":\"dev\",\"name\":\"newtype_derive\",\"req\":\"^0.1\"},{\"kind\":\"dev\",\"name\":\"odds\",\"req\":\"^0.2.26\"},{\"default_features\":false,\"kind\":\"dev\",\"name\":\"quickcheck\",\"req\":\"^0.5\"},{\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.3.10\"},{\"name\":\"unchecked-index\",\"optional\":true,\"req\":\"^0.2.2\"}],\"features\":{\"all\":[\"jetscii\",\"pcmp\",\"pattern\",\"test-set\"],\"benchmarks\":[\"galil-seiferas\",\"pattern\",\"unchecked-index\"],\"default\":[\"use_std\"],\"pattern\":[],\"pcmp\":[\"unchecked-index\"],\"test-set\":[],\"use_std\":[\"memchr/use_std\"]}}", "typed-builder-macro_0.20.1": "{\"dependencies\":[{\"name\":\"proc-macro2\",\"req\":\"^1\"},{\"name\":\"quote\",\"req\":\"^1\"},{\"features\":[\"full\",\"extra-traits\"],\"name\":\"syn\",\"req\":\"^2\"}],\"features\":{}}", "typed-builder_0.20.1": "{\"dependencies\":[{\"name\":\"typed-builder-macro\",\"req\":\"=0.20.1\"}],\"features\":{}}", "typed-path_0.12.3": "{\"dependencies\":[],\"features\":{\"default\":[\"std\"],\"std\":[]}}", "typenum_1.19.0": "{\"dependencies\":[{\"default_features\":false,\"name\":\"scale-info\",\"optional\":true,\"req\":\"^1.0\"}],\"features\":{\"const-generics\":[],\"force_unix_path_separator\":[],\"i128\":[],\"no_std\":[],\"scale_info\":[\"scale-info/derive\"],\"strict\":[]}}", - "typespec_1.0.0": "{\"dependencies\":[{\"name\":\"base64\",\"req\":\"^0.22\"},{\"name\":\"bytes\",\"req\":\"^1.11.1\"},{\"name\":\"futures\",\"req\":\"^0.3\"},{\"features\":[\"serialize\",\"serde-types\"],\"name\":\"quick-xml\",\"optional\":true,\"req\":\"^0.39.0\"},{\"features\":[\"derive\"],\"name\":\"serde\",\"optional\":true,\"req\":\"^1.0\"},{\"name\":\"serde_json\",\"optional\":true,\"req\":\"^1.0.149\"},{\"kind\":\"dev\",\"name\":\"thiserror\",\"req\":\"^2.0\"},{\"name\":\"url\",\"req\":\"^2.5\"}],\"features\":{\"default\":[\"http\",\"json\"],\"http\":[],\"json\":[\"dep:serde\",\"dep:serde_json\"],\"xml\":[\"dep:serde\",\"dep:quick-xml\"]}}", + "typespec_1.1.0": "{\"dependencies\":[{\"name\":\"base64\",\"req\":\"^0.22\"},{\"name\":\"bytes\",\"req\":\"^1.11.1\"},{\"name\":\"futures\",\"req\":\"^0.3\"},{\"features\":[\"serialize\",\"serde-types\"],\"name\":\"quick-xml\",\"optional\":true,\"req\":\"^0.41.0\"},{\"features\":[\"derive\"],\"name\":\"serde\",\"optional\":true,\"req\":\"^1.0\"},{\"name\":\"serde_json\",\"optional\":true,\"req\":\"^1.0.149\"},{\"kind\":\"dev\",\"name\":\"thiserror\",\"req\":\"^2.0\"},{\"name\":\"url\",\"req\":\"^2.5\"}],\"features\":{\"default\":[\"http\",\"json\"],\"http\":[],\"json\":[\"dep:serde\",\"dep:serde_json\"],\"xml\":[\"dep:serde\",\"dep:quick-xml\"]}}", "typespec_client_core_1.0.0": "{\"dependencies\":[{\"name\":\"async-trait\",\"req\":\"^0.1\"},{\"name\":\"base64\",\"req\":\"^0.22\"},{\"name\":\"bytes\",\"req\":\"^1.11.1\"},{\"name\":\"dyn-clone\",\"req\":\"^1.0\"},{\"name\":\"futures\",\"req\":\"^0.3\"},{\"name\":\"gloo-timers\",\"optional\":true,\"req\":\"^0.3\"},{\"name\":\"pin-project\",\"req\":\"^1.1\"},{\"features\":[\"sys_rng\"],\"name\":\"rand\",\"req\":\"^0.10.1\"},{\"default_features\":false,\"features\":[\"stream\"],\"name\":\"reqwest\",\"optional\":true,\"req\":\"^0.13.2\"},{\"name\":\"rust_decimal\",\"optional\":true,\"req\":\"^1.40.0\"},{\"features\":[\"derive\"],\"name\":\"serde\",\"req\":\"^1.0\"},{\"name\":\"serde_json\",\"optional\":true,\"req\":\"^1.0.149\"},{\"features\":[\"serde-well-known\",\"macros\"],\"name\":\"time\",\"req\":\"^0.3.47\"},{\"default_features\":false,\"features\":[\"macros\",\"time\",\"macros\",\"rt-multi-thread\",\"time\"],\"name\":\"tokio\",\"optional\":true,\"req\":\"^1.49\"},{\"default_features\":false,\"features\":[\"macros\",\"time\",\"fs\"],\"kind\":\"dev\",\"name\":\"tokio\",\"req\":\"^1.49\"},{\"name\":\"tracing\",\"req\":\"^0.1.44\"},{\"kind\":\"dev\",\"name\":\"tracing\",\"req\":\"^0.1.44\"},{\"kind\":\"dev\",\"name\":\"tracing-subscriber\",\"req\":\"^0.3\"},{\"default_features\":false,\"name\":\"typespec\",\"req\":\"^1.0.0\"},{\"name\":\"typespec_macros\",\"optional\":true,\"req\":\"^1.0.0\"},{\"kind\":\"dev\",\"name\":\"typespec_macros\",\"req\":\"^1.0.0\"},{\"name\":\"url\",\"req\":\"^2.5\"},{\"features\":[\"v4\"],\"name\":\"uuid\",\"req\":\"^1.20\"}],\"features\":{\"debug\":[\"typespec_macros?/debug\"],\"decimal\":[\"dep:rust_decimal\"],\"default\":[\"http\",\"json\",\"reqwest\",\"reqwest_deflate\",\"reqwest_gzip\",\"reqwest_rustls\",\"tokio\"],\"derive\":[\"dep:typespec_macros\"],\"http\":[\"typespec/http\"],\"json\":[\"dep:serde_json\",\"typespec/json\"],\"reqwest\":[\"dep:reqwest\"],\"reqwest_deflate\":[\"reqwest\",\"reqwest/deflate\"],\"reqwest_gzip\":[\"reqwest\",\"reqwest/gzip\"],\"reqwest_rustls\":[\"reqwest\",\"reqwest/rustls\"],\"test\":[],\"tokio\":[\"tokio/sync\",\"tokio/time\"],\"xml\":[\"dep:serde_json\",\"typespec/xml\"]}}", "typespec_macros_1.0.0": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"cargo_metadata\",\"req\":\"^0.23.1\"},{\"name\":\"proc-macro2\",\"req\":\"^1.0.106\"},{\"name\":\"quote\",\"req\":\"^1.0.44\"},{\"name\":\"rustc_version\",\"req\":\"^0.4\"},{\"features\":[\"derive\"],\"kind\":\"dev\",\"name\":\"serde\",\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"serde_json\",\"req\":\"^1.0.149\"},{\"features\":[\"full\"],\"name\":\"syn\",\"req\":\"^2.0.115\"},{\"default_features\":false,\"features\":[\"macros\",\"time\"],\"kind\":\"dev\",\"name\":\"tokio\",\"req\":\"^1.49\"}],\"features\":{\"debug\":[]}}", "ucd-trie_0.1.7": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"once_cell\",\"req\":\"^1\"}],\"features\":{\"default\":[\"std\"],\"std\":[]}}", "unicase_2.8.1": "{\"dependencies\":[],\"features\":{\"nightly\":[]}}", + "unicase_2.9.0": "{\"dependencies\":[],\"features\":{\"nightly\":[]}}", "unicode-bidi_0.3.18": "{\"dependencies\":[{\"name\":\"flame\",\"optional\":true,\"req\":\"^0.2\"},{\"name\":\"flamer\",\"optional\":true,\"req\":\"^0.4\"},{\"default_features\":false,\"features\":[\"derive\"],\"name\":\"serde\",\"optional\":true,\"req\":\">=0.8, <2.0\"},{\"kind\":\"dev\",\"name\":\"serde_test\",\"req\":\">=0.8, <2.0\"},{\"features\":[\"union\"],\"name\":\"smallvec\",\"optional\":true,\"req\":\">=1.13\"}],\"features\":{\"bench_it\":[],\"default\":[\"std\",\"hardcoded-data\"],\"flame_it\":[\"flame\",\"flamer\"],\"hardcoded-data\":[],\"std\":[],\"unstable\":[],\"with_serde\":[\"serde\"]}}", "unicode-ident_1.0.24": "{\"dependencies\":[{\"default_features\":false,\"kind\":\"dev\",\"name\":\"criterion\",\"req\":\"^0.8\",\"target\":\"cfg(not(miri))\"},{\"kind\":\"dev\",\"name\":\"fst\",\"req\":\"^0.4\"},{\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.10\"},{\"kind\":\"dev\",\"name\":\"roaring\",\"req\":\"^0.11\"},{\"default_features\":false,\"kind\":\"dev\",\"name\":\"ucd-trie\",\"req\":\"^0.1\"},{\"kind\":\"dev\",\"name\":\"unicode-xid\",\"req\":\"^0.2.6\"}],\"features\":{}}", "unicode-normalization_0.1.24": "{\"dependencies\":[{\"features\":[\"alloc\"],\"name\":\"tinyvec\",\"req\":\"^1\"}],\"features\":{\"default\":[\"std\"],\"std\":[]}}", "unicode-properties_0.1.3": "{\"dependencies\":[],\"features\":{\"default\":[\"general-category\",\"emoji\"],\"emoji\":[],\"general-category\":[]}}", "unicode-xid_0.2.6": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"criterion\",\"req\":\"^0.3\"}],\"features\":{\"bench\":[],\"default\":[],\"no_std\":[]}}", "untrusted_0.9.0": "{\"dependencies\":[],\"features\":{}}", + "ureq_2.12.1": "{\"dependencies\":[{\"name\":\"base64\",\"req\":\"^0.22\"},{\"name\":\"brotli-decompressor\",\"optional\":true,\"req\":\"^4.0.0\"},{\"default_features\":false,\"name\":\"cookie\",\"optional\":true,\"req\":\"^0.18\"},{\"default_features\":false,\"features\":[\"preserve_order\",\"serde_json\"],\"name\":\"cookie_store\",\"optional\":true,\"req\":\"^0.21.1\"},{\"name\":\"encoding_rs\",\"optional\":true,\"req\":\"^0.8\"},{\"default_features\":false,\"features\":[\"humantime\"],\"kind\":\"dev\",\"name\":\"env_logger\",\"req\":\"<=0.9\"},{\"name\":\"flate2\",\"optional\":true,\"req\":\"^1.0.22\"},{\"name\":\"hootbin\",\"optional\":true,\"req\":\"^0.1.5\"},{\"name\":\"http\",\"optional\":true,\"req\":\"^1.1\"},{\"name\":\"http-02\",\"optional\":true,\"package\":\"http\",\"req\":\"^0.2\"},{\"name\":\"log\",\"req\":\"^0.4\"},{\"name\":\"native-tls\",\"optional\":true,\"req\":\"^0.2\"},{\"name\":\"once_cell\",\"req\":\"^1\"},{\"default_features\":false,\"features\":[\"ring\",\"logging\",\"std\",\"tls12\"],\"name\":\"rustls\",\"optional\":true,\"req\":\"^0.23.19\"},{\"default_features\":false,\"features\":[\"std\",\"ring\"],\"kind\":\"dev\",\"name\":\"rustls\",\"req\":\"^0.23.5\"},{\"name\":\"rustls-native-certs\",\"optional\":true,\"req\":\"^0.7\"},{\"kind\":\"dev\",\"name\":\"rustls-pemfile\",\"req\":\"^2.0\"},{\"name\":\"rustls-pki-types\",\"optional\":true,\"req\":\"^1\"},{\"name\":\"serde\",\"optional\":true,\"req\":\"^1\"},{\"features\":[\"derive\"],\"kind\":\"dev\",\"name\":\"serde\",\"req\":\"^1\"},{\"name\":\"serde_json\",\"optional\":true,\"req\":\"^1.0.97\"},{\"name\":\"socks\",\"optional\":true,\"req\":\"^0.3\"},{\"name\":\"url\",\"req\":\"^2.5.0\"},{\"name\":\"webpki-roots\",\"optional\":true,\"req\":\"^0.26\"}],\"features\":{\"brotli\":[\"dep:brotli-decompressor\"],\"charset\":[\"dep:encoding_rs\"],\"cookies\":[\"dep:cookie\",\"dep:cookie_store\"],\"default\":[\"tls\",\"gzip\"],\"gzip\":[\"dep:flate2\"],\"http-crate\":[\"dep:http\"],\"http-interop\":[\"dep:http-02\"],\"json\":[\"dep:serde\",\"dep:serde_json\"],\"native-certs\":[\"dep:rustls-native-certs\"],\"native-tls\":[\"dep:native-tls\"],\"proxy-from-env\":[],\"socks-proxy\":[\"dep:socks\"],\"testdeps\":[\"dep:hootbin\"],\"tls\":[\"dep:webpki-roots\",\"dep:rustls\",\"dep:rustls-pki-types\"]}}", "url_2.5.8": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"bencher\",\"req\":\"^0.1\"},{\"default_features\":false,\"features\":[\"alloc\"],\"name\":\"form_urlencoded\",\"req\":\"^1.2.2\"},{\"default_features\":false,\"features\":[\"alloc\",\"compiled_data\"],\"name\":\"idna\",\"req\":\"^1.1.0\"},{\"default_features\":false,\"features\":[\"alloc\"],\"name\":\"percent-encoding\",\"req\":\"^2.3.2\"},{\"default_features\":false,\"name\":\"serde\",\"optional\":true,\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"serde\",\"req\":\"^1.0\"},{\"default_features\":false,\"name\":\"serde_derive\",\"optional\":true,\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"serde_derive\",\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"serde_json\",\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"wasm-bindgen-test\",\"req\":\"^0.3\",\"target\":\"cfg(all(target_arch = \\\"wasm32\\\", target_os = \\\"unknown\\\"))\"}],\"features\":{\"debugger_visualizer\":[],\"default\":[\"std\"],\"expose_internals\":[],\"serde\":[\"dep:serde\",\"dep:serde_derive\"],\"std\":[\"idna/std\",\"percent-encoding/std\",\"form_urlencoded/std\",\"serde?/std\"]}}", "urlencoding_2.1.3": "{\"dependencies\":[],\"features\":{}}", "utf8-width_0.1.7": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"bencher\",\"req\":\"^0.1.5\"}],\"features\":{}}", @@ -1327,30 +1287,46 @@ "version_check_0.9.5": "{\"dependencies\":[],\"features\":{}}", "vsimd_0.8.0": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"const-str\",\"req\":\"^0.5.3\"},{\"features\":[\"js\"],\"kind\":\"dev\",\"name\":\"getrandom\",\"req\":\"^0.2.8\",\"target\":\"cfg(target_arch = \\\"wasm32\\\")\"},{\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.8.5\"},{\"kind\":\"dev\",\"name\":\"wasm-bindgen-test\",\"req\":\"^0.3.33\",\"target\":\"cfg(target_arch = \\\"wasm32\\\")\"}],\"features\":{\"alloc\":[],\"detect\":[\"std\"],\"std\":[\"alloc\"],\"unstable\":[]}}", "walkdir_2.5.0": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"doc-comment\",\"req\":\"^0.3\"},{\"name\":\"same-file\",\"req\":\"^1.0.1\"},{\"name\":\"winapi-util\",\"req\":\"^0.1.1\",\"target\":\"cfg(windows)\"}],\"features\":{}}", + "walrus-macro_0.24.0": "{\"dependencies\":[{\"name\":\"heck\",\"req\":\"^0.5.0\"},{\"name\":\"proc-macro2\",\"req\":\"^1.0.86\"},{\"name\":\"quote\",\"req\":\"^1.0.37\"},{\"features\":[\"extra-traits\"],\"name\":\"syn\",\"req\":\"^2.0.77\"}],\"features\":{}}", + "walrus_0.24.5": "{\"dependencies\":[{\"name\":\"anyhow\",\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"criterion\",\"req\":\"^0.5.0\"},{\"kind\":\"dev\",\"name\":\"env_logger\",\"req\":\"^0.11.0\"},{\"name\":\"gimli\",\"req\":\"^0.26.0\"},{\"name\":\"id-arena\",\"req\":\"^2.2.1\"},{\"name\":\"leb128\",\"req\":\"^0.2.4\"},{\"name\":\"log\",\"req\":\"^0.4.8\"},{\"name\":\"rayon\",\"optional\":true,\"req\":\"^1.1.0\"},{\"name\":\"walrus-macro\",\"req\":\"=0.24.0\"},{\"name\":\"wasm-encoder\",\"req\":\"^0.240.0\"},{\"name\":\"wasmparser\",\"req\":\"^0.240.0\"}],\"features\":{\"parallel\":[\"rayon\",\"id-arena/rayon\"]}}", "want_0.3.1": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"tokio-executor\",\"req\":\"^0.2.0-alpha.2\"},{\"kind\":\"dev\",\"name\":\"tokio-sync\",\"req\":\"^0.2.0-alpha.2\"},{\"name\":\"try-lock\",\"req\":\"^0.2.4\"}],\"features\":{}}", "wasi_0.11.1+wasi-snapshot-preview1": "{\"dependencies\":[{\"name\":\"core\",\"optional\":true,\"package\":\"rustc-std-workspace-core\",\"req\":\"^1.0\"},{\"name\":\"rustc-std-workspace-alloc\",\"optional\":true,\"req\":\"^1.0\"}],\"features\":{\"default\":[\"std\"],\"rustc-dep-of-std\":[\"core\",\"rustc-std-workspace-alloc\"],\"std\":[]}}", "wasip2_1.0.2+wasi-0.2.9": "{\"dependencies\":[{\"name\":\"alloc\",\"optional\":true,\"package\":\"rustc-std-workspace-alloc\",\"req\":\"^1.0\"},{\"name\":\"core\",\"optional\":true,\"package\":\"rustc-std-workspace-core\",\"req\":\"^1.0\"},{\"default_features\":false,\"name\":\"wit-bindgen\",\"req\":\"^0.51.0\"}],\"features\":{\"bitflags\":[\"wit-bindgen/bitflags\"],\"default\":[\"std\",\"bitflags\"],\"rustc-dep-of-std\":[\"core\",\"alloc\",\"wit-bindgen/rustc-dep-of-std\"],\"std\":[]}}", "wasip2_1.0.3+wasi-0.2.9": "{\"dependencies\":[{\"name\":\"alloc\",\"optional\":true,\"package\":\"rustc-std-workspace-alloc\",\"req\":\"^1.0\"},{\"name\":\"core\",\"optional\":true,\"package\":\"rustc-std-workspace-core\",\"req\":\"^1.0\"},{\"default_features\":false,\"name\":\"wit-bindgen\",\"req\":\"^0.57.1\"}],\"features\":{\"bitflags\":[\"wit-bindgen/bitflags\"],\"default\":[\"std\",\"bitflags\"],\"rustc-dep-of-std\":[\"core\",\"alloc\",\"wit-bindgen/rustc-dep-of-std\"],\"std\":[]}}", "wasip3_0.4.0+wasi-0.3.0-rc-2026-01-06": "{\"dependencies\":[{\"name\":\"bytes\",\"optional\":true,\"req\":\"^1.10.1\"},{\"kind\":\"dev\",\"name\":\"futures\",\"req\":\"^0.3.31\"},{\"name\":\"http\",\"optional\":true,\"req\":\"^1.3.1\"},{\"kind\":\"dev\",\"name\":\"http\",\"req\":\"^1.3.1\"},{\"name\":\"http-body\",\"optional\":true,\"req\":\"^1.0.1\"},{\"name\":\"thiserror\",\"optional\":true,\"req\":\"^2.0.17\"},{\"default_features\":false,\"features\":[\"async\"],\"name\":\"wit-bindgen\",\"req\":\"^0.51.0\"},{\"default_features\":false,\"features\":[\"async-spawn\"],\"kind\":\"dev\",\"name\":\"wit-bindgen\",\"req\":\"^0.51.0\"}],\"features\":{\"http-compat\":[\"dep:bytes\",\"dep:http-body\",\"dep:http\",\"dep:thiserror\",\"wit-bindgen/async-spawn\"]}}", + "wasm-bindgen-cli-support_0.2.105": "{\"dependencies\":[{\"name\":\"anyhow\",\"req\":\"^1.0\"},{\"name\":\"base64\",\"req\":\"^0.22\"},{\"name\":\"leb128\",\"req\":\"^0.2\"},{\"name\":\"log\",\"req\":\"^0.4\"},{\"kind\":\"dev\",\"name\":\"rstest\",\"req\":\"^0.26\"},{\"name\":\"rustc-demangle\",\"req\":\"^0.1.13\"},{\"features\":[\"derive\"],\"name\":\"serde\",\"req\":\"^1.0\"},{\"name\":\"serde_json\",\"req\":\"^1.0\"},{\"features\":[\"parallel\"],\"name\":\"walrus\",\"req\":\"^0.24.2\"},{\"name\":\"wasm-bindgen-shared\",\"req\":\"=0.2.105\"},{\"name\":\"wasmparser\",\"req\":\"^0.214\"},{\"kind\":\"dev\",\"name\":\"wasmprinter\",\"req\":\"^0.214\"},{\"kind\":\"dev\",\"name\":\"wast\",\"req\":\"^214\"},{\"kind\":\"dev\",\"name\":\"wat\",\"req\":\"^1.0\"}],\"features\":{}}", + "wasm-bindgen-cli_0.2.105": "{\"dependencies\":[{\"name\":\"anyhow\",\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"assert_cmd\",\"req\":\"^2\"},{\"features\":[\"derive\"],\"name\":\"clap\",\"req\":\"^4\"},{\"name\":\"env_logger\",\"req\":\"^0.11.5\"},{\"name\":\"log\",\"req\":\"^0.4\"},{\"default_features\":false,\"name\":\"native-tls\",\"optional\":true,\"req\":\"^0.2\"},{\"kind\":\"dev\",\"name\":\"predicates\",\"req\":\"^3\"},{\"kind\":\"dev\",\"name\":\"pretty_assertions\",\"req\":\"^1.4\"},{\"kind\":\"dev\",\"name\":\"regex\",\"req\":\"^1.11.2\"},{\"default_features\":false,\"name\":\"rouille\",\"req\":\"^3.0.0\"},{\"kind\":\"dev\",\"name\":\"rstest\",\"req\":\"^0.26\"},{\"features\":[\"derive\"],\"name\":\"serde\",\"req\":\"^1.0\"},{\"name\":\"serde_derive\",\"req\":\"^1.0\"},{\"name\":\"serde_json\",\"req\":\"^1.0\"},{\"name\":\"shlex\",\"req\":\"^1\"},{\"name\":\"tempfile\",\"req\":\"^3.0\"},{\"default_features\":false,\"features\":[\"brotli\",\"gzip\"],\"name\":\"ureq\",\"req\":\"^2.7\"},{\"name\":\"walrus\",\"req\":\"^0.24.2\"},{\"name\":\"wasm-bindgen-cli-support\",\"req\":\"=0.2.105\"},{\"kind\":\"dev\",\"name\":\"wasmparser\",\"req\":\"^0.214\"},{\"kind\":\"dev\",\"name\":\"wasmprinter\",\"req\":\"^0.214\"}],\"features\":{\"default\":[\"rustls-tls\"],\"native-tls\":[\"ureq/native-tls\"],\"openssl\":[\"dep:native-tls\"],\"rustls-tls\":[\"ureq/tls\"],\"vendored-openssl\":[\"openssl\",\"native-tls/vendored\"]}}", + "wasm-bindgen-futures_0.4.55": "{\"dependencies\":[{\"name\":\"cfg-if\",\"req\":\"^1.0.0\"},{\"kind\":\"dev\",\"name\":\"futures-channel\",\"req\":\"^0.3\",\"target\":\"cfg(target_arch = \\\"wasm32\\\")\"},{\"default_features\":false,\"name\":\"futures-core\",\"optional\":true,\"req\":\"^0.3.8\"},{\"default_features\":false,\"kind\":\"dev\",\"name\":\"futures-lite\",\"req\":\"^2\",\"target\":\"cfg(target_arch = \\\"wasm32\\\")\"},{\"default_features\":false,\"name\":\"js-sys\",\"req\":\"=0.3.82\"},{\"default_features\":false,\"name\":\"once_cell\",\"req\":\"^1.12\"},{\"default_features\":false,\"name\":\"wasm-bindgen\",\"req\":\"=0.2.105\"},{\"default_features\":false,\"features\":[\"MessageEvent\",\"Worker\"],\"name\":\"web-sys\",\"req\":\"=0.3.82\",\"target\":\"cfg(target_feature = \\\"atomics\\\")\"}],\"features\":{\"default\":[\"std\"],\"futures-core-03-stream\":[\"futures-core\"],\"std\":[\"wasm-bindgen/std\",\"js-sys/std\",\"web-sys/std\"]}}", "wasm-bindgen-futures_0.4.75": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"futures-channel\",\"req\":\"^0.3\",\"target\":\"cfg(target_arch = \\\"wasm32\\\")\"},{\"default_features\":false,\"kind\":\"dev\",\"name\":\"futures-lite\",\"req\":\"^2\",\"target\":\"cfg(target_arch = \\\"wasm32\\\")\"},{\"default_features\":false,\"name\":\"js-sys\",\"req\":\"=0.3.102\"},{\"default_features\":false,\"name\":\"wasm-bindgen\",\"req\":\"=0.2.125\"}],\"features\":{\"default\":[\"std\"],\"futures-core-03-stream\":[\"js-sys/futures-core-03-stream\"],\"std\":[\"wasm-bindgen/std\",\"js-sys/std\"]}}", + "wasm-bindgen-macro-support_0.2.105": "{\"dependencies\":[{\"name\":\"bumpalo\",\"req\":\"^3.0.0\"},{\"name\":\"proc-macro2\",\"req\":\"^1.0\"},{\"name\":\"quote\",\"req\":\"^1.0\"},{\"features\":[\"visit\",\"visit-mut\",\"full\"],\"name\":\"syn\",\"req\":\"^2.0\"},{\"name\":\"wasm-bindgen-shared\",\"req\":\"=0.2.105\"}],\"features\":{\"extra-traits\":[\"syn/extra-traits\"],\"strict-macro\":[]}}", "wasm-bindgen-macro-support_0.2.125": "{\"dependencies\":[{\"name\":\"bumpalo\",\"req\":\"^3.0.0\"},{\"name\":\"proc-macro2\",\"req\":\"^1.0\"},{\"name\":\"quote\",\"req\":\"^1.0\"},{\"features\":[\"visit\",\"visit-mut\",\"full\",\"extra-traits\"],\"name\":\"syn\",\"req\":\"^2.0\"},{\"name\":\"wasm-bindgen-shared\",\"req\":\"=0.2.125\"}],\"features\":{\"extra-traits\":[\"syn/extra-traits\"],\"strict-macro\":[]}}", + "wasm-bindgen-macro_0.2.105": "{\"dependencies\":[{\"name\":\"quote\",\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"trybuild\",\"req\":\"^1.0\"},{\"name\":\"wasm-bindgen-macro-support\",\"req\":\"=0.2.105\"}],\"features\":{\"strict-macro\":[\"wasm-bindgen-macro-support/strict-macro\"]}}", "wasm-bindgen-macro_0.2.125": "{\"dependencies\":[{\"name\":\"quote\",\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"trybuild\",\"req\":\"^1.0\"},{\"name\":\"wasm-bindgen-macro-support\",\"req\":\"=0.2.125\"}],\"features\":{\"strict-macro\":[\"wasm-bindgen-macro-support/strict-macro\"]}}", + "wasm-bindgen-shared_0.2.105": "{\"dependencies\":[{\"name\":\"unicode-ident\",\"req\":\"^1.0.5\"}],\"features\":{}}", "wasm-bindgen-shared_0.2.125": "{\"dependencies\":[{\"name\":\"unicode-ident\",\"req\":\"^1.0.5\"}],\"features\":{}}", + "wasm-bindgen-test-macro_0.3.55": "{\"dependencies\":[{\"name\":\"proc-macro2\",\"req\":\"^1.0\"},{\"name\":\"quote\",\"req\":\"^1.0\"},{\"default_features\":false,\"features\":[\"parsing\",\"proc-macro\",\"derive\",\"printing\"],\"name\":\"syn\",\"req\":\"^2.0\"},{\"features\":[\"macros\",\"rt-multi-thread\"],\"kind\":\"dev\",\"name\":\"tokio\",\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"trybuild\",\"req\":\"^1.0\"}],\"features\":{}}", + "wasm-bindgen-test_0.3.55": "{\"dependencies\":[{\"name\":\"gg-alloc\",\"optional\":true,\"req\":\"^1.0\"},{\"default_features\":false,\"name\":\"js-sys\",\"req\":\"=0.3.82\"},{\"name\":\"minicov\",\"req\":\"^0.3\",\"target\":\"cfg(all(target_arch = \\\"wasm32\\\", wasm_bindgen_unstable_test_coverage))\"},{\"default_features\":false,\"name\":\"wasm-bindgen\",\"req\":\"=0.2.105\"},{\"default_features\":false,\"name\":\"wasm-bindgen-futures\",\"req\":\"=0.4.55\"},{\"name\":\"wasm-bindgen-test-macro\",\"req\":\"=0.3.55\"}],\"features\":{\"default\":[\"std\"],\"std\":[\"wasm-bindgen/std\",\"js-sys/std\",\"wasm-bindgen-futures/std\"]}}", + "wasm-bindgen_0.2.105": "{\"dependencies\":[{\"name\":\"cfg-if\",\"req\":\"^1.0.0\"},{\"default_features\":false,\"name\":\"once_cell\",\"req\":\"^1.12\"},{\"kind\":\"dev\",\"name\":\"once_cell\",\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"paste\",\"req\":\"^1\",\"target\":\"cfg(target_arch = \\\"wasm32\\\")\"},{\"kind\":\"build\",\"name\":\"rustversion-compat\",\"package\":\"rustversion\",\"req\":\"^1.0\"},{\"name\":\"serde\",\"optional\":true,\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"serde_derive\",\"req\":\"^1.0\",\"target\":\"cfg(target_arch = \\\"wasm32\\\")\"},{\"name\":\"serde_json\",\"optional\":true,\"req\":\"^1.0\"},{\"name\":\"wasm-bindgen-macro\",\"req\":\"=0.2.105\"},{\"name\":\"wasm-bindgen-shared\",\"req\":\"=0.2.105\"}],\"features\":{\"default\":[\"std\"],\"enable-interning\":[\"std\"],\"gg-alloc\":[],\"msrv\":[],\"rustversion\":[],\"serde-serialize\":[\"serde\",\"serde_json\",\"std\"],\"spans\":[],\"std\":[],\"strict-macro\":[\"wasm-bindgen-macro/strict-macro\"],\"xxx_debug_only_print_generated_code\":[]}}", "wasm-bindgen_0.2.125": "{\"dependencies\":[{\"name\":\"cfg-if\",\"req\":\"^1.0.0\"},{\"default_features\":false,\"name\":\"once_cell\",\"req\":\"^1.12\"},{\"kind\":\"dev\",\"name\":\"once_cell\",\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"paste\",\"req\":\"^1\",\"target\":\"cfg(target_arch = \\\"wasm32\\\")\"},{\"kind\":\"dev\",\"name\":\"paste\",\"req\":\"^1\",\"target\":\"cfg(target_arch = \\\"wasm64\\\")\"},{\"kind\":\"build\",\"name\":\"rustversion-compat\",\"package\":\"rustversion\",\"req\":\"^1.0.6\"},{\"name\":\"serde\",\"optional\":true,\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"serde_derive\",\"req\":\"^1.0\",\"target\":\"cfg(target_arch = \\\"wasm32\\\")\"},{\"kind\":\"dev\",\"name\":\"serde_derive\",\"req\":\"^1.0\",\"target\":\"cfg(target_arch = \\\"wasm64\\\")\"},{\"name\":\"serde_json\",\"optional\":true,\"req\":\"^1.0\"},{\"name\":\"wasm-bindgen-macro\",\"req\":\"=0.2.125\"},{\"name\":\"wasm-bindgen-shared\",\"req\":\"=0.2.125\"}],\"features\":{\"default\":[\"std\"],\"enable-interning\":[\"std\"],\"gg-alloc\":[],\"msrv\":[],\"rustversion\":[],\"serde-serialize\":[\"serde\",\"serde_json\",\"std\"],\"spans\":[],\"std\":[],\"strict-macro\":[\"wasm-bindgen-macro/strict-macro\"],\"xxx_debug_only_print_generated_code\":[]}}", + "wasm-encoder_0.240.0": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"anyhow\",\"req\":\"^1.0.58\"},{\"default_features\":false,\"name\":\"leb128fmt\",\"req\":\"^0.1.0\"},{\"kind\":\"dev\",\"name\":\"tempfile\",\"req\":\"^3.2.0\"},{\"default_features\":false,\"features\":[\"simd\",\"simd\"],\"name\":\"wasmparser\",\"optional\":true,\"req\":\"^0.240.0\"},{\"default_features\":false,\"kind\":\"dev\",\"name\":\"wasmprinter\",\"req\":\"^0.240.0\"}],\"features\":{\"component-model\":[\"wasmparser?/component-model\"],\"default\":[\"std\",\"component-model\"],\"std\":[\"wasmparser?/std\"]}}", "wasm-encoder_0.244.0": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"anyhow\",\"req\":\"^1.0.58\"},{\"default_features\":false,\"name\":\"leb128fmt\",\"req\":\"^0.1.0\"},{\"kind\":\"dev\",\"name\":\"tempfile\",\"req\":\"^3.2.0\"},{\"default_features\":false,\"features\":[\"simd\",\"simd\"],\"name\":\"wasmparser\",\"optional\":true,\"req\":\"^0.244.0\"},{\"default_features\":false,\"kind\":\"dev\",\"name\":\"wasmprinter\",\"req\":\"^0.244.0\"}],\"features\":{\"component-model\":[\"wasmparser?/component-model\"],\"default\":[\"std\",\"component-model\"],\"std\":[\"wasmparser?/std\"]}}", "wasm-metadata_0.244.0": "{\"dependencies\":[{\"name\":\"anyhow\",\"req\":\"^1.0.58\"},{\"name\":\"auditable-serde\",\"optional\":true,\"req\":\"^0.8.0\"},{\"features\":[\"derive\"],\"name\":\"clap\",\"optional\":true,\"req\":\"^4.0.0\"},{\"name\":\"flate2\",\"optional\":true,\"req\":\"^1.1.0\"},{\"default_features\":false,\"features\":[\"serde\"],\"name\":\"indexmap\",\"req\":\"^2.7.0\"},{\"default_features\":false,\"features\":[\"alloc\"],\"name\":\"serde\",\"optional\":true,\"req\":\"^1.0.166\"},{\"name\":\"serde_derive\",\"optional\":true,\"req\":\"^1.0.166\"},{\"name\":\"serde_json\",\"optional\":true,\"req\":\"^1\"},{\"name\":\"spdx\",\"optional\":true,\"req\":\"^0.10.1\"},{\"name\":\"url\",\"optional\":true,\"req\":\"^2.0.0\"},{\"default_features\":false,\"features\":[\"std\",\"component-model\"],\"name\":\"wasm-encoder\",\"req\":\"^0.244.0\"},{\"default_features\":false,\"features\":[\"simd\",\"std\",\"component-model\",\"hash-collections\"],\"name\":\"wasmparser\",\"req\":\"^0.244.0\"}],\"features\":{\"default\":[\"oci\",\"serde\"],\"oci\":[\"dep:auditable-serde\",\"dep:flate2\",\"dep:url\",\"dep:spdx\",\"dep:serde_json\",\"serde\"],\"serde\":[\"dep:serde_derive\",\"dep:serde\"]}}", "wasm-streams_0.5.0": "{\"dependencies\":[{\"features\":[\"io\",\"sink\"],\"name\":\"futures-util\",\"req\":\"^0.3.31\"},{\"features\":[\"futures\"],\"kind\":\"dev\",\"name\":\"gloo-timers\",\"req\":\"^0.3.0\"},{\"name\":\"js-sys\",\"req\":\"^0.3.85\"},{\"kind\":\"dev\",\"name\":\"pin-project\",\"req\":\"^1\"},{\"features\":[\"macros\",\"rt\"],\"kind\":\"dev\",\"name\":\"tokio\",\"req\":\"^1\"},{\"name\":\"wasm-bindgen\",\"req\":\"^0.2.108\"},{\"name\":\"wasm-bindgen-futures\",\"req\":\"^0.4.58\"},{\"kind\":\"dev\",\"name\":\"wasm-bindgen-test\",\"req\":\"^0.3.58\"},{\"features\":[\"AbortSignal\",\"QueuingStrategy\",\"ReadableStream\",\"ReadableStreamType\",\"ReadableWritablePair\",\"ReadableStreamByobReader\",\"ReadableStreamReaderMode\",\"ReadableStreamReadResult\",\"ReadableStreamByobRequest\",\"ReadableStreamDefaultReader\",\"ReadableByteStreamController\",\"ReadableStreamGetReaderOptions\",\"ReadableStreamDefaultController\",\"StreamPipeOptions\",\"TransformStream\",\"TransformStreamDefaultController\",\"Transformer\",\"UnderlyingSink\",\"UnderlyingSource\",\"WritableStream\",\"WritableStreamDefaultController\",\"WritableStreamDefaultWriter\"],\"name\":\"web-sys\",\"req\":\"^0.3.85\"},{\"features\":[\"console\",\"AbortSignal\",\"ErrorEvent\",\"PromiseRejectionEvent\",\"Response\",\"ReadableStream\",\"Window\"],\"kind\":\"dev\",\"name\":\"web-sys\",\"req\":\"^0.3.85\"}],\"features\":{}}", + "wasmparser_0.214.0": "{\"dependencies\":[{\"default_features\":false,\"name\":\"ahash\",\"optional\":true,\"req\":\"^0.8.11\"},{\"kind\":\"dev\",\"name\":\"anyhow\",\"req\":\"^1.0.58\"},{\"name\":\"bitflags\",\"req\":\"^2.4.1\"},{\"default_features\":false,\"kind\":\"dev\",\"name\":\"criterion\",\"req\":\"^0.5.1\"},{\"kind\":\"dev\",\"name\":\"env_logger\",\"req\":\"^0.11\"},{\"default_features\":false,\"features\":[\"ahash\"],\"name\":\"hashbrown\",\"optional\":true,\"req\":\"^0.14.3\"},{\"default_features\":false,\"name\":\"indexmap\",\"optional\":true,\"req\":\"^2.0.0\"},{\"kind\":\"dev\",\"name\":\"log\",\"req\":\"^0.4.17\"},{\"kind\":\"dev\",\"name\":\"once_cell\",\"req\":\"^1.13.0\"},{\"kind\":\"dev\",\"name\":\"rayon\",\"req\":\"^1.3\"},{\"default_features\":false,\"name\":\"semver\",\"optional\":true,\"req\":\"^1.0.0\"},{\"default_features\":false,\"features\":[\"alloc\"],\"name\":\"serde\",\"optional\":true,\"req\":\"^1.0.166\"}],\"features\":{\"default\":[\"std\",\"validate\",\"serde\"],\"no-hash-maps\":[],\"serde\":[\"dep:serde\",\"indexmap/serde\",\"hashbrown/serde\"],\"std\":[\"indexmap/std\"],\"validate\":[\"dep:indexmap\",\"dep:semver\",\"dep:hashbrown\",\"dep:ahash\"]}}", + "wasmparser_0.240.0": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"anyhow\",\"req\":\"^1.0.58\"},{\"name\":\"bitflags\",\"req\":\"^2.4.1\"},{\"default_features\":false,\"kind\":\"dev\",\"name\":\"criterion\",\"req\":\"^0.5.1\"},{\"kind\":\"dev\",\"name\":\"env_logger\",\"req\":\"^0.11\"},{\"default_features\":false,\"features\":[\"default-hasher\"],\"name\":\"hashbrown\",\"optional\":true,\"req\":\"^0.15.2\"},{\"default_features\":false,\"name\":\"indexmap\",\"optional\":true,\"req\":\"^2.7.0\"},{\"kind\":\"dev\",\"name\":\"log\",\"req\":\"^0.4.17\"},{\"kind\":\"dev\",\"name\":\"once_cell\",\"req\":\"^1.13.0\"},{\"kind\":\"dev\",\"name\":\"rayon\",\"req\":\"^1.3\"},{\"default_features\":false,\"name\":\"semver\",\"optional\":true,\"req\":\"^1.0.0\"},{\"default_features\":false,\"features\":[\"alloc\"],\"name\":\"serde\",\"optional\":true,\"req\":\"^1.0.166\"}],\"features\":{\"component-model\":[\"dep:semver\"],\"default\":[\"std\",\"validate\",\"serde\",\"features\",\"component-model\",\"hash-collections\",\"simd\"],\"features\":[],\"hash-collections\":[\"dep:hashbrown\",\"dep:indexmap\"],\"prefer-btree-collections\":[],\"serde\":[\"dep:serde\",\"indexmap?/serde\",\"hashbrown?/serde\"],\"simd\":[],\"std\":[\"indexmap?/std\"],\"validate\":[]}}", "wasmparser_0.244.0": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"anyhow\",\"req\":\"^1.0.58\"},{\"name\":\"bitflags\",\"req\":\"^2.4.1\"},{\"default_features\":false,\"kind\":\"dev\",\"name\":\"criterion\",\"req\":\"^0.5.1\"},{\"kind\":\"dev\",\"name\":\"env_logger\",\"req\":\"^0.11\"},{\"default_features\":false,\"features\":[\"default-hasher\"],\"name\":\"hashbrown\",\"optional\":true,\"req\":\"^0.15.2\"},{\"default_features\":false,\"name\":\"indexmap\",\"optional\":true,\"req\":\"^2.7.0\"},{\"kind\":\"dev\",\"name\":\"log\",\"req\":\"^0.4.17\"},{\"kind\":\"dev\",\"name\":\"once_cell\",\"req\":\"^1.13.0\"},{\"kind\":\"dev\",\"name\":\"rayon\",\"req\":\"^1.3\"},{\"default_features\":false,\"name\":\"semver\",\"optional\":true,\"req\":\"^1.0.0\"},{\"default_features\":false,\"features\":[\"alloc\"],\"name\":\"serde\",\"optional\":true,\"req\":\"^1.0.166\"}],\"features\":{\"component-model\":[\"dep:semver\"],\"default\":[\"std\",\"validate\",\"serde\",\"features\",\"component-model\",\"hash-collections\",\"simd\"],\"features\":[],\"hash-collections\":[\"dep:hashbrown\",\"dep:indexmap\"],\"prefer-btree-collections\":[],\"serde\":[\"dep:serde\",\"indexmap?/serde\",\"hashbrown?/serde\"],\"simd\":[],\"std\":[\"indexmap?/std\"],\"validate\":[]}}", "web-sys_0.3.102": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"futures\",\"req\":\"^0.3\",\"target\":\"cfg(target_arch = \\\"wasm32\\\")\"},{\"default_features\":false,\"name\":\"js-sys\",\"req\":\"=0.3.102\"},{\"default_features\":false,\"name\":\"wasm-bindgen\",\"req\":\"=0.2.125\"}],\"features\":{\"AbortController\":[],\"AbortSignal\":[\"EventTarget\"],\"AbstractRange\":[],\"AddEventListenerOptions\":[],\"AesCbcParams\":[],\"AesCtrParams\":[],\"AesDerivedKeyParams\":[],\"AesGcmParams\":[],\"AesKeyAlgorithm\":[],\"AesKeyGenParams\":[],\"Algorithm\":[],\"AlignSetting\":[],\"AllowedBluetoothDevice\":[],\"AllowedUsbDevice\":[],\"AlphaOption\":[],\"AnalyserNode\":[\"AudioNode\",\"EventTarget\"],\"AnalyserOptions\":[],\"AngleInstancedArrays\":[],\"Animation\":[\"EventTarget\"],\"AnimationEffect\":[],\"AnimationEvent\":[\"Event\"],\"AnimationEventInit\":[],\"AnimationPlayState\":[],\"AnimationPlaybackEvent\":[\"Event\"],\"AnimationPlaybackEventInit\":[],\"AnimationPropertyDetails\":[],\"AnimationPropertyValueDetails\":[],\"AnimationTimeline\":[],\"AssignedNodesOptions\":[],\"AttestationConveyancePreference\":[],\"Attr\":[\"EventTarget\",\"Node\"],\"AttributeNameValue\":[],\"AudioBuffer\":[],\"AudioBufferOptions\":[],\"AudioBufferSourceNode\":[\"AudioNode\",\"AudioScheduledSourceNode\",\"EventTarget\"],\"AudioBufferSourceOptions\":[],\"AudioConfiguration\":[],\"AudioContext\":[\"BaseAudioContext\",\"EventTarget\"],\"AudioContextLatencyCategory\":[],\"AudioContextOptions\":[],\"AudioContextState\":[],\"AudioData\":[],\"AudioDataCopyToOptions\":[],\"AudioDataInit\":[],\"AudioDecoder\":[\"EventTarget\"],\"AudioDecoderConfig\":[],\"AudioDecoderInit\":[],\"AudioDecoderSupport\":[],\"AudioDestinationNode\":[\"AudioNode\",\"EventTarget\"],\"AudioEncoder\":[\"EventTarget\"],\"AudioEncoderConfig\":[],\"AudioEncoderInit\":[],\"AudioEncoderSupport\":[],\"AudioListener\":[],\"AudioNode\":[\"EventTarget\"],\"AudioNodeOptions\":[],\"AudioParam\":[],\"AudioParamMap\":[],\"AudioProcessingEvent\":[\"Event\"],\"AudioSampleFormat\":[],\"AudioScheduledSourceNode\":[\"AudioNode\",\"EventTarget\"],\"AudioSinkInfo\":[],\"AudioSinkOptions\":[],\"AudioSinkType\":[],\"AudioStreamTrack\":[\"EventTarget\",\"MediaStreamTrack\"],\"AudioTrack\":[],\"AudioTrackList\":[\"EventTarget\"],\"AudioWorklet\":[\"Worklet\"],\"AudioWorkletGlobalScope\":[\"WorkletGlobalScope\"],\"AudioWorkletNode\":[\"AudioNode\",\"EventTarget\"],\"AudioWorkletNodeOptions\":[],\"AudioWorkletProcessor\":[],\"AuthenticationExtensionsClientInputs\":[],\"AuthenticationExtensionsClientInputsJson\":[],\"AuthenticationExtensionsClientOutputs\":[],\"AuthenticationExtensionsClientOutputsJson\":[],\"AuthenticationExtensionsDevicePublicKeyInputs\":[],\"AuthenticationExtensionsDevicePublicKeyOutputs\":[],\"AuthenticationExtensionsLargeBlobInputs\":[],\"AuthenticationExtensionsLargeBlobOutputs\":[],\"AuthenticationExtensionsPrfInputs\":[],\"AuthenticationExtensionsPrfOutputs\":[],\"AuthenticationExtensionsPrfValues\":[],\"AuthenticationResponseJson\":[],\"AuthenticatorAssertionResponse\":[\"AuthenticatorResponse\"],\"AuthenticatorAssertionResponseJson\":[],\"AuthenticatorAttachment\":[],\"AuthenticatorAttestationResponse\":[\"AuthenticatorResponse\"],\"AuthenticatorAttestationResponseJson\":[],\"AuthenticatorResponse\":[],\"AuthenticatorSelectionCriteria\":[],\"AuthenticatorTransport\":[],\"AutoKeyword\":[],\"AutocompleteInfo\":[],\"BarProp\":[],\"BaseAudioContext\":[\"EventTarget\"],\"BaseComputedKeyframe\":[],\"BaseKeyframe\":[],\"BasePropertyIndexedKeyframe\":[],\"BasicCardRequest\":[],\"BasicCardResponse\":[],\"BasicCardType\":[],\"BatteryManager\":[\"EventTarget\"],\"BeforeUnloadEvent\":[\"Event\"],\"BinaryType\":[],\"BiquadFilterNode\":[\"AudioNode\",\"EventTarget\"],\"BiquadFilterOptions\":[],\"BiquadFilterType\":[],\"BitrateMode\":[],\"Blob\":[],\"BlobEvent\":[\"Event\"],\"BlobEventInit\":[],\"BlobPropertyBag\":[],\"BlockParsingOptions\":[],\"Bluetooth\":[\"EventTarget\"],\"BluetoothAdvertisingEvent\":[\"Event\"],\"BluetoothAdvertisingEventInit\":[],\"BluetoothCharacteristicProperties\":[],\"BluetoothDataFilterInit\":[],\"BluetoothDevice\":[\"EventTarget\"],\"BluetoothLeScanFilterInit\":[],\"BluetoothManufacturerDataMap\":[],\"BluetoothPermissionDescriptor\":[],\"BluetoothPermissionResult\":[\"EventTarget\",\"PermissionStatus\"],\"BluetoothPermissionStorage\":[],\"BluetoothRemoteGattCharacteristic\":[\"EventTarget\"],\"BluetoothRemoteGattDescriptor\":[],\"BluetoothRemoteGattServer\":[],\"BluetoothRemoteGattService\":[\"EventTarget\"],\"BluetoothServiceDataMap\":[],\"BluetoothUuid\":[],\"BoxQuadOptions\":[],\"BroadcastChannel\":[\"EventTarget\"],\"BrowserElementDownloadOptions\":[],\"BrowserElementExecuteScriptOptions\":[],\"BrowserFeedWriter\":[],\"BrowserFindCaseSensitivity\":[],\"BrowserFindDirection\":[],\"ByteLengthQueuingStrategy\":[],\"Cache\":[],\"CacheBatchOperation\":[],\"CacheQueryOptions\":[],\"CacheStorage\":[],\"CacheStorageNamespace\":[],\"CanvasCaptureMediaStream\":[\"EventTarget\",\"MediaStream\"],\"CanvasCaptureMediaStreamTrack\":[\"EventTarget\",\"MediaStreamTrack\"],\"CanvasGradient\":[],\"CanvasPattern\":[],\"CanvasRenderingContext2d\":[],\"CanvasWindingRule\":[],\"CaretChangedReason\":[],\"CaretPosition\":[],\"CaretStateChangedEventInit\":[],\"CdataSection\":[\"CharacterData\",\"EventTarget\",\"Node\",\"Text\"],\"ChannelCountMode\":[],\"ChannelInterpretation\":[],\"ChannelMergerNode\":[\"AudioNode\",\"EventTarget\"],\"ChannelMergerOptions\":[],\"ChannelSplitterNode\":[\"AudioNode\",\"EventTarget\"],\"ChannelSplitterOptions\":[],\"CharacterData\":[\"EventTarget\",\"Node\"],\"CheckerboardReason\":[],\"CheckerboardReport\":[],\"CheckerboardReportService\":[],\"ChromeFilePropertyBag\":[],\"ChromeWorker\":[\"EventTarget\",\"Worker\"],\"Client\":[],\"ClientQueryOptions\":[],\"ClientRectsAndTexts\":[],\"ClientType\":[],\"Clients\":[],\"Clipboard\":[\"EventTarget\"],\"ClipboardEvent\":[\"Event\"],\"ClipboardEventInit\":[],\"ClipboardItem\":[],\"ClipboardItemOptions\":[],\"ClipboardPermissionDescriptor\":[],\"ClipboardUnsanitizedFormats\":[],\"CloseEvent\":[\"Event\"],\"CloseEventInit\":[],\"CodecState\":[],\"CollectedClientData\":[],\"ColorSpaceConversion\":[],\"CommandEvent\":[\"Event\"],\"CommandEventInit\":[],\"Comment\":[\"CharacterData\",\"EventTarget\",\"Node\"],\"CompositeOperation\":[],\"CompositionEvent\":[\"Event\",\"UiEvent\"],\"CompositionEventInit\":[],\"CompressionFormat\":[],\"CompressionStream\":[],\"ComputedEffectTiming\":[],\"ConnStatusDict\":[],\"ConnectionType\":[],\"ConsoleCounter\":[],\"ConsoleCounterError\":[],\"ConsoleEvent\":[],\"ConsoleInstance\":[],\"ConsoleInstanceOptions\":[],\"ConsoleLevel\":[],\"ConsoleLogLevel\":[],\"ConsoleProfileEvent\":[],\"ConsoleStackEntry\":[],\"ConsoleTimerError\":[],\"ConsoleTimerLogOrEnd\":[],\"ConsoleTimerStart\":[],\"ConstantSourceNode\":[\"AudioNode\",\"AudioScheduledSourceNode\",\"EventTarget\"],\"ConstantSourceOptions\":[],\"ConstrainBooleanParameters\":[],\"ConstrainDomStringParameters\":[],\"ConstrainDoubleRange\":[],\"ConstrainLongRange\":[],\"ContextAttributes2d\":[],\"ConvertCoordinateOptions\":[],\"ConvolverNode\":[\"AudioNode\",\"EventTarget\"],\"ConvolverOptions\":[],\"CookieChangeEvent\":[\"Event\"],\"CookieChangeEventInit\":[],\"CookieInit\":[],\"CookieListItem\":[],\"CookieSameSite\":[],\"CookieStore\":[\"EventTarget\"],\"CookieStoreDeleteOptions\":[],\"CookieStoreGetOptions\":[],\"CookieStoreManager\":[],\"Coordinates\":[],\"CountQueuingStrategy\":[],\"Credential\":[],\"CredentialCreationOptions\":[],\"CredentialPropertiesOutput\":[],\"CredentialRequestOptions\":[],\"CredentialsContainer\":[],\"Crypto\":[],\"CryptoKey\":[],\"CryptoKeyPair\":[],\"CssAnimation\":[\"Animation\",\"EventTarget\"],\"CssBoxType\":[],\"CssConditionRule\":[\"CssGroupingRule\",\"CssRule\"],\"CssCounterStyleRule\":[\"CssRule\"],\"CssFontFaceRule\":[\"CssRule\"],\"CssFontFeatureValuesRule\":[\"CssRule\"],\"CssGroupingRule\":[\"CssRule\"],\"CssImportRule\":[\"CssRule\"],\"CssKeyframeRule\":[\"CssRule\"],\"CssKeyframesRule\":[\"CssRule\"],\"CssMediaRule\":[\"CssConditionRule\",\"CssGroupingRule\",\"CssRule\"],\"CssNamespaceRule\":[\"CssRule\"],\"CssPageRule\":[\"CssRule\"],\"CssPseudoElement\":[],\"CssRule\":[],\"CssRuleList\":[],\"CssStyleDeclaration\":[],\"CssStyleRule\":[\"CssRule\"],\"CssStyleSheet\":[\"StyleSheet\"],\"CssStyleSheetParsingMode\":[],\"CssSupportsRule\":[\"CssConditionRule\",\"CssGroupingRule\",\"CssRule\"],\"CssTransition\":[\"Animation\",\"EventTarget\"],\"CssViewTransitionRule\":[\"CssRule\"],\"CustomElementRegistry\":[],\"CustomEvent\":[\"Event\"],\"CustomEventInit\":[],\"DataTransfer\":[],\"DataTransferItem\":[],\"DataTransferItemList\":[],\"DateTimeValue\":[],\"DecoderDoctorNotification\":[],\"DecoderDoctorNotificationType\":[],\"DecompressionStream\":[],\"DedicatedWorkerGlobalScope\":[\"EventTarget\",\"WorkerGlobalScope\"],\"DelayNode\":[\"AudioNode\",\"EventTarget\"],\"DelayOptions\":[],\"DeviceAcceleration\":[],\"DeviceAccelerationInit\":[],\"DeviceLightEvent\":[\"Event\"],\"DeviceLightEventInit\":[],\"DeviceMotionEvent\":[\"Event\"],\"DeviceMotionEventInit\":[],\"DeviceOrientationEvent\":[\"Event\"],\"DeviceOrientationEventInit\":[],\"DeviceProximityEvent\":[\"Event\"],\"DeviceProximityEventInit\":[],\"DeviceRotationRate\":[],\"DeviceRotationRateInit\":[],\"DhKeyDeriveParams\":[],\"DirectionSetting\":[],\"Directory\":[],\"DirectoryPickerOptions\":[],\"DisplayMediaStreamConstraints\":[],\"DisplayNameOptions\":[],\"DisplayNameResult\":[],\"DistanceModelType\":[],\"DnsCacheDict\":[],\"DnsCacheEntry\":[],\"DnsLookupDict\":[],\"Document\":[\"EventTarget\",\"Node\"],\"DocumentFragment\":[\"EventTarget\",\"Node\"],\"DocumentTimeline\":[\"AnimationTimeline\"],\"DocumentTimelineOptions\":[],\"DocumentType\":[\"EventTarget\",\"Node\"],\"DomError\":[],\"DomException\":[],\"DomImplementation\":[],\"DomMatrix\":[\"DomMatrixReadOnly\"],\"DomMatrix2dInit\":[],\"DomMatrixInit\":[],\"DomMatrixReadOnly\":[],\"DomParser\":[],\"DomPoint\":[\"DomPointReadOnly\"],\"DomPointInit\":[],\"DomPointReadOnly\":[],\"DomQuad\":[],\"DomQuadInit\":[],\"DomQuadJson\":[],\"DomRect\":[\"DomRectReadOnly\"],\"DomRectInit\":[],\"DomRectList\":[],\"DomRectReadOnly\":[],\"DomRequest\":[\"EventTarget\"],\"DomRequestReadyState\":[],\"DomStringList\":[],\"DomStringMap\":[],\"DomTokenList\":[],\"DomWindowResizeEventDetail\":[],\"DoubleRange\":[],\"DragEvent\":[\"Event\",\"MouseEvent\",\"UiEvent\"],\"DragEventInit\":[],\"DynamicsCompressorNode\":[\"AudioNode\",\"EventTarget\"],\"DynamicsCompressorOptions\":[],\"EcKeyAlgorithm\":[],\"EcKeyGenParams\":[],\"EcKeyImportParams\":[],\"EcdhKeyDeriveParams\":[],\"EcdsaParams\":[],\"EffectTiming\":[],\"Element\":[\"EventTarget\",\"Node\"],\"ElementCreationOptions\":[],\"ElementDefinitionOptions\":[],\"EncodedAudioChunk\":[],\"EncodedAudioChunkInit\":[],\"EncodedAudioChunkMetadata\":[],\"EncodedAudioChunkType\":[],\"EncodedVideoChunk\":[],\"EncodedVideoChunkInit\":[],\"EncodedVideoChunkMetadata\":[],\"EncodedVideoChunkType\":[],\"EndingTypes\":[],\"ErrorCallback\":[],\"ErrorEvent\":[\"Event\"],\"ErrorEventInit\":[],\"Event\":[],\"EventInit\":[],\"EventListener\":[],\"EventListenerOptions\":[],\"EventModifierInit\":[],\"EventSource\":[\"EventTarget\"],\"EventSourceInit\":[],\"EventTarget\":[],\"Exception\":[],\"ExtBlendMinmax\":[],\"ExtColorBufferFloat\":[],\"ExtColorBufferHalfFloat\":[],\"ExtDisjointTimerQuery\":[],\"ExtFragDepth\":[],\"ExtSRgb\":[],\"ExtShaderTextureLod\":[],\"ExtTextureFilterAnisotropic\":[],\"ExtTextureNorm16\":[],\"ExtendableCookieChangeEvent\":[\"Event\",\"ExtendableEvent\"],\"ExtendableCookieChangeEventInit\":[],\"ExtendableEvent\":[\"Event\"],\"ExtendableEventInit\":[],\"ExtendableMessageEvent\":[\"Event\",\"ExtendableEvent\"],\"ExtendableMessageEventInit\":[],\"External\":[],\"FakePluginMimeEntry\":[],\"FakePluginTagInit\":[],\"FetchEvent\":[\"Event\",\"ExtendableEvent\"],\"FetchEventInit\":[],\"FetchObserver\":[\"EventTarget\"],\"FetchReadableStreamReadDataArray\":[],\"FetchReadableStreamReadDataDone\":[],\"FetchState\":[],\"File\":[\"Blob\"],\"FileCallback\":[],\"FileList\":[],\"FilePickerAcceptType\":[],\"FilePickerOptions\":[],\"FilePropertyBag\":[],\"FileReader\":[\"EventTarget\"],\"FileReaderSync\":[],\"FileSystem\":[],\"FileSystemCreateWritableOptions\":[],\"FileSystemDirectoryEntry\":[\"FileSystemEntry\"],\"FileSystemDirectoryHandle\":[\"FileSystemHandle\"],\"FileSystemDirectoryReader\":[],\"FileSystemEntriesCallback\":[],\"FileSystemEntry\":[],\"FileSystemEntryCallback\":[],\"FileSystemFileEntry\":[\"FileSystemEntry\"],\"FileSystemFileHandle\":[\"FileSystemHandle\"],\"FileSystemFlags\":[],\"FileSystemGetDirectoryOptions\":[],\"FileSystemGetFileOptions\":[],\"FileSystemHandle\":[],\"FileSystemHandleKind\":[],\"FileSystemHandlePermissionDescriptor\":[],\"FileSystemPermissionDescriptor\":[],\"FileSystemPermissionMode\":[],\"FileSystemReadWriteOptions\":[],\"FileSystemRemoveOptions\":[],\"FileSystemSyncAccessHandle\":[],\"FileSystemSyncAccessHandleMode\":[],\"FileSystemSyncAccessHandleOptions\":[],\"FileSystemWritableFileStream\":[\"WritableStream\"],\"FillLightMode\":[],\"FillMode\":[],\"FlashClassification\":[],\"FlowControlType\":[],\"FocusEvent\":[\"Event\",\"UiEvent\"],\"FocusEventInit\":[],\"FocusOptions\":[],\"FontData\":[],\"FontFace\":[],\"FontFaceDescriptors\":[],\"FontFaceLoadStatus\":[],\"FontFaceSet\":[\"EventTarget\"],\"FontFaceSetIterator\":[],\"FontFaceSetIteratorResult\":[],\"FontFaceSetLoadEvent\":[\"Event\"],\"FontFaceSetLoadEventInit\":[],\"FontFaceSetLoadStatus\":[],\"FormData\":[],\"FrameType\":[],\"FuzzingFunctions\":[],\"GainNode\":[\"AudioNode\",\"EventTarget\"],\"GainOptions\":[],\"Gamepad\":[],\"GamepadButton\":[],\"GamepadEffectParameters\":[],\"GamepadEvent\":[\"Event\"],\"GamepadEventInit\":[],\"GamepadHand\":[],\"GamepadHapticActuator\":[],\"GamepadHapticActuatorType\":[],\"GamepadHapticEffectType\":[],\"GamepadHapticsResult\":[],\"GamepadMappingType\":[],\"GamepadPose\":[],\"GamepadTouch\":[],\"Geolocation\":[],\"GeolocationCoordinates\":[],\"GeolocationPosition\":[],\"GeolocationPositionError\":[],\"GestureEvent\":[\"Event\",\"UiEvent\"],\"GetAnimationsOptions\":[],\"GetRootNodeOptions\":[],\"GetUserMediaRequest\":[],\"Gpu\":[],\"GpuAdapter\":[],\"GpuAdapterInfo\":[],\"GpuAddressMode\":[],\"GpuAutoLayoutMode\":[],\"GpuBindGroup\":[],\"GpuBindGroupDescriptor\":[],\"GpuBindGroupEntry\":[],\"GpuBindGroupLayout\":[],\"GpuBindGroupLayoutDescriptor\":[],\"GpuBindGroupLayoutEntry\":[],\"GpuBlendComponent\":[],\"GpuBlendFactor\":[],\"GpuBlendOperation\":[],\"GpuBlendState\":[],\"GpuBuffer\":[],\"GpuBufferBinding\":[],\"GpuBufferBindingLayout\":[],\"GpuBufferBindingType\":[],\"GpuBufferDescriptor\":[],\"GpuBufferMapState\":[],\"GpuCanvasAlphaMode\":[],\"GpuCanvasConfiguration\":[],\"GpuCanvasContext\":[],\"GpuCanvasToneMapping\":[],\"GpuCanvasToneMappingMode\":[],\"GpuColorDict\":[],\"GpuColorTargetState\":[],\"GpuCommandBuffer\":[],\"GpuCommandBufferDescriptor\":[],\"GpuCommandEncoder\":[],\"GpuCommandEncoderDescriptor\":[],\"GpuCompareFunction\":[],\"GpuCompilationInfo\":[],\"GpuCompilationMessage\":[],\"GpuCompilationMessageType\":[],\"GpuComputePassDescriptor\":[],\"GpuComputePassEncoder\":[],\"GpuComputePassTimestampWrites\":[],\"GpuComputePipeline\":[],\"GpuComputePipelineDescriptor\":[],\"GpuCopyExternalImageDestInfo\":[],\"GpuCopyExternalImageSourceInfo\":[],\"GpuCullMode\":[],\"GpuDepthStencilState\":[],\"GpuDevice\":[\"EventTarget\"],\"GpuDeviceDescriptor\":[],\"GpuDeviceLostInfo\":[],\"GpuDeviceLostReason\":[],\"GpuError\":[],\"GpuErrorFilter\":[],\"GpuExtent3dDict\":[],\"GpuExternalTexture\":[],\"GpuExternalTextureBindingLayout\":[],\"GpuExternalTextureDescriptor\":[],\"GpuFeatureName\":[],\"GpuFilterMode\":[],\"GpuFragmentState\":[],\"GpuFrontFace\":[],\"GpuIndexFormat\":[],\"GpuInternalError\":[\"GpuError\"],\"GpuLoadOp\":[],\"GpuMipmapFilterMode\":[],\"GpuMultisampleState\":[],\"GpuObjectDescriptorBase\":[],\"GpuOrigin2dDict\":[],\"GpuOrigin3dDict\":[],\"GpuOutOfMemoryError\":[\"GpuError\"],\"GpuPipelineDescriptorBase\":[],\"GpuPipelineError\":[\"DomException\"],\"GpuPipelineErrorInit\":[],\"GpuPipelineErrorReason\":[],\"GpuPipelineLayout\":[],\"GpuPipelineLayoutDescriptor\":[],\"GpuPowerPreference\":[],\"GpuPrimitiveState\":[],\"GpuPrimitiveTopology\":[],\"GpuProgrammableStage\":[],\"GpuQuerySet\":[],\"GpuQuerySetDescriptor\":[],\"GpuQueryType\":[],\"GpuQueue\":[],\"GpuQueueDescriptor\":[],\"GpuRenderBundle\":[],\"GpuRenderBundleDescriptor\":[],\"GpuRenderBundleEncoder\":[],\"GpuRenderBundleEncoderDescriptor\":[],\"GpuRenderPassColorAttachment\":[],\"GpuRenderPassDepthStencilAttachment\":[],\"GpuRenderPassDescriptor\":[],\"GpuRenderPassEncoder\":[],\"GpuRenderPassLayout\":[],\"GpuRenderPassTimestampWrites\":[],\"GpuRenderPipeline\":[],\"GpuRenderPipelineDescriptor\":[],\"GpuRequestAdapterOptions\":[],\"GpuSampler\":[],\"GpuSamplerBindingLayout\":[],\"GpuSamplerBindingType\":[],\"GpuSamplerDescriptor\":[],\"GpuShaderModule\":[],\"GpuShaderModuleCompilationHint\":[],\"GpuShaderModuleDescriptor\":[],\"GpuStencilFaceState\":[],\"GpuStencilOperation\":[],\"GpuStorageTextureAccess\":[],\"GpuStorageTextureBindingLayout\":[],\"GpuStoreOp\":[],\"GpuSupportedFeatures\":[],\"GpuSupportedLimits\":[],\"GpuTexelCopyBufferInfo\":[],\"GpuTexelCopyBufferLayout\":[],\"GpuTexelCopyTextureInfo\":[],\"GpuTexture\":[],\"GpuTextureAspect\":[],\"GpuTextureBindingLayout\":[],\"GpuTextureDescriptor\":[],\"GpuTextureDimension\":[],\"GpuTextureFormat\":[],\"GpuTextureSampleType\":[],\"GpuTextureView\":[],\"GpuTextureViewDescriptor\":[],\"GpuTextureViewDimension\":[],\"GpuUncapturedErrorEvent\":[\"Event\"],\"GpuUncapturedErrorEventInit\":[],\"GpuValidationError\":[\"GpuError\"],\"GpuVertexAttribute\":[],\"GpuVertexBufferLayout\":[],\"GpuVertexFormat\":[],\"GpuVertexState\":[],\"GpuVertexStepMode\":[],\"GroupedHistoryEventInit\":[],\"HalfOpenInfoDict\":[],\"HardwareAcceleration\":[],\"HashChangeEvent\":[\"Event\"],\"HashChangeEventInit\":[],\"Headers\":[],\"HeadersGuardEnum\":[],\"Hid\":[\"EventTarget\"],\"HidCollectionInfo\":[],\"HidConnectionEvent\":[\"Event\"],\"HidConnectionEventInit\":[],\"HidDevice\":[\"EventTarget\"],\"HidDeviceFilter\":[],\"HidDeviceRequestOptions\":[],\"HidInputReportEvent\":[\"Event\"],\"HidInputReportEventInit\":[],\"HidReportInfo\":[],\"HidReportItem\":[],\"HidUnitSystem\":[],\"HiddenPluginEventInit\":[],\"Highlight\":[],\"HighlightHitResult\":[],\"HighlightRegistry\":[],\"HighlightType\":[],\"HighlightsFromPointOptions\":[],\"History\":[],\"HitRegionOptions\":[],\"HkdfParams\":[],\"HmacDerivedKeyParams\":[],\"HmacImportParams\":[],\"HmacKeyAlgorithm\":[],\"HmacKeyGenParams\":[],\"HtmlAllCollection\":[],\"HtmlAnchorElement\":[\"Element\",\"EventTarget\",\"HtmlElement\",\"Node\"],\"HtmlAreaElement\":[\"Element\",\"EventTarget\",\"HtmlElement\",\"Node\"],\"HtmlAudioElement\":[\"Element\",\"EventTarget\",\"HtmlElement\",\"HtmlMediaElement\",\"Node\"],\"HtmlBaseElement\":[\"Element\",\"EventTarget\",\"HtmlElement\",\"Node\"],\"HtmlBodyElement\":[\"Element\",\"EventTarget\",\"HtmlElement\",\"Node\"],\"HtmlBrElement\":[\"Element\",\"EventTarget\",\"HtmlElement\",\"Node\"],\"HtmlButtonElement\":[\"Element\",\"EventTarget\",\"HtmlElement\",\"Node\"],\"HtmlCanvasElement\":[\"Element\",\"EventTarget\",\"HtmlElement\",\"Node\"],\"HtmlCollection\":[],\"HtmlDListElement\":[\"Element\",\"EventTarget\",\"HtmlElement\",\"Node\"],\"HtmlDataElement\":[\"Element\",\"EventTarget\",\"HtmlElement\",\"Node\"],\"HtmlDataListElement\":[\"Element\",\"EventTarget\",\"HtmlElement\",\"Node\"],\"HtmlDetailsElement\":[\"Element\",\"EventTarget\",\"HtmlElement\",\"Node\"],\"HtmlDialogElement\":[\"Element\",\"EventTarget\",\"HtmlElement\",\"Node\"],\"HtmlDirectoryElement\":[\"Element\",\"EventTarget\",\"HtmlElement\",\"Node\"],\"HtmlDivElement\":[\"Element\",\"EventTarget\",\"HtmlElement\",\"Node\"],\"HtmlDocument\":[\"Document\",\"EventTarget\",\"Node\"],\"HtmlElement\":[\"Element\",\"EventTarget\",\"Node\"],\"HtmlEmbedElement\":[\"Element\",\"EventTarget\",\"HtmlElement\",\"Node\"],\"HtmlFieldSetElement\":[\"Element\",\"EventTarget\",\"HtmlElement\",\"Node\"],\"HtmlFontElement\":[\"Element\",\"EventTarget\",\"HtmlElement\",\"Node\"],\"HtmlFormControlsCollection\":[\"HtmlCollection\"],\"HtmlFormElement\":[\"Element\",\"EventTarget\",\"HtmlElement\",\"Node\"],\"HtmlFrameElement\":[\"Element\",\"EventTarget\",\"HtmlElement\",\"Node\"],\"HtmlFrameSetElement\":[\"Element\",\"EventTarget\",\"HtmlElement\",\"Node\"],\"HtmlHeadElement\":[\"Element\",\"EventTarget\",\"HtmlElement\",\"Node\"],\"HtmlHeadingElement\":[\"Element\",\"EventTarget\",\"HtmlElement\",\"Node\"],\"HtmlHrElement\":[\"Element\",\"EventTarget\",\"HtmlElement\",\"Node\"],\"HtmlHtmlElement\":[\"Element\",\"EventTarget\",\"HtmlElement\",\"Node\"],\"HtmlIFrameElement\":[\"Element\",\"EventTarget\",\"HtmlElement\",\"Node\"],\"HtmlImageElement\":[\"Element\",\"EventTarget\",\"HtmlElement\",\"Node\"],\"HtmlInputElement\":[\"Element\",\"EventTarget\",\"HtmlElement\",\"Node\"],\"HtmlLabelElement\":[\"Element\",\"EventTarget\",\"HtmlElement\",\"Node\"],\"HtmlLegendElement\":[\"Element\",\"EventTarget\",\"HtmlElement\",\"Node\"],\"HtmlLiElement\":[\"Element\",\"EventTarget\",\"HtmlElement\",\"Node\"],\"HtmlLinkElement\":[\"Element\",\"EventTarget\",\"HtmlElement\",\"Node\"],\"HtmlMapElement\":[\"Element\",\"EventTarget\",\"HtmlElement\",\"Node\"],\"HtmlMediaElement\":[\"Element\",\"EventTarget\",\"HtmlElement\",\"Node\"],\"HtmlMenuElement\":[\"Element\",\"EventTarget\",\"HtmlElement\",\"Node\"],\"HtmlMenuItemElement\":[\"Element\",\"EventTarget\",\"HtmlElement\",\"Node\"],\"HtmlMetaElement\":[\"Element\",\"EventTarget\",\"HtmlElement\",\"Node\"],\"HtmlMeterElement\":[\"Element\",\"EventTarget\",\"HtmlElement\",\"Node\"],\"HtmlModElement\":[\"Element\",\"EventTarget\",\"HtmlElement\",\"Node\"],\"HtmlOListElement\":[\"Element\",\"EventTarget\",\"HtmlElement\",\"Node\"],\"HtmlObjectElement\":[\"Element\",\"EventTarget\",\"HtmlElement\",\"Node\"],\"HtmlOptGroupElement\":[\"Element\",\"EventTarget\",\"HtmlElement\",\"Node\"],\"HtmlOptionElement\":[\"Element\",\"EventTarget\",\"HtmlElement\",\"Node\"],\"HtmlOptionsCollection\":[\"HtmlCollection\"],\"HtmlOutputElement\":[\"Element\",\"EventTarget\",\"HtmlElement\",\"Node\"],\"HtmlParagraphElement\":[\"Element\",\"EventTarget\",\"HtmlElement\",\"Node\"],\"HtmlParamElement\":[\"Element\",\"EventTarget\",\"HtmlElement\",\"Node\"],\"HtmlPictureElement\":[\"Element\",\"EventTarget\",\"HtmlElement\",\"Node\"],\"HtmlPreElement\":[\"Element\",\"EventTarget\",\"HtmlElement\",\"Node\"],\"HtmlProgressElement\":[\"Element\",\"EventTarget\",\"HtmlElement\",\"Node\"],\"HtmlQuoteElement\":[\"Element\",\"EventTarget\",\"HtmlElement\",\"Node\"],\"HtmlScriptElement\":[\"Element\",\"EventTarget\",\"HtmlElement\",\"Node\"],\"HtmlSelectElement\":[\"Element\",\"EventTarget\",\"HtmlElement\",\"Node\"],\"HtmlSlotElement\":[\"Element\",\"EventTarget\",\"HtmlElement\",\"Node\"],\"HtmlSourceElement\":[\"Element\",\"EventTarget\",\"HtmlElement\",\"Node\"],\"HtmlSpanElement\":[\"Element\",\"EventTarget\",\"HtmlElement\",\"Node\"],\"HtmlStyleElement\":[\"Element\",\"EventTarget\",\"HtmlElement\",\"Node\"],\"HtmlTableCaptionElement\":[\"Element\",\"EventTarget\",\"HtmlElement\",\"Node\"],\"HtmlTableCellElement\":[\"Element\",\"EventTarget\",\"HtmlElement\",\"Node\"],\"HtmlTableColElement\":[\"Element\",\"EventTarget\",\"HtmlElement\",\"Node\"],\"HtmlTableElement\":[\"Element\",\"EventTarget\",\"HtmlElement\",\"Node\"],\"HtmlTableRowElement\":[\"Element\",\"EventTarget\",\"HtmlElement\",\"Node\"],\"HtmlTableSectionElement\":[\"Element\",\"EventTarget\",\"HtmlElement\",\"Node\"],\"HtmlTemplateElement\":[\"Element\",\"EventTarget\",\"HtmlElement\",\"Node\"],\"HtmlTextAreaElement\":[\"Element\",\"EventTarget\",\"HtmlElement\",\"Node\"],\"HtmlTimeElement\":[\"Element\",\"EventTarget\",\"HtmlElement\",\"Node\"],\"HtmlTitleElement\":[\"Element\",\"EventTarget\",\"HtmlElement\",\"Node\"],\"HtmlTrackElement\":[\"Element\",\"EventTarget\",\"HtmlElement\",\"Node\"],\"HtmlUListElement\":[\"Element\",\"EventTarget\",\"HtmlElement\",\"Node\"],\"HtmlUnknownElement\":[\"Element\",\"EventTarget\",\"HtmlElement\",\"Node\"],\"HtmlVideoElement\":[\"Element\",\"EventTarget\",\"HtmlElement\",\"HtmlMediaElement\",\"Node\"],\"HttpConnDict\":[],\"HttpConnInfo\":[],\"HttpConnectionElement\":[],\"IdbCursor\":[],\"IdbCursorDirection\":[],\"IdbCursorWithValue\":[\"IdbCursor\"],\"IdbDatabase\":[\"EventTarget\"],\"IdbFactory\":[],\"IdbFileHandle\":[\"EventTarget\"],\"IdbFileMetadataParameters\":[],\"IdbFileRequest\":[\"DomRequest\",\"EventTarget\"],\"IdbIndex\":[],\"IdbIndexParameters\":[],\"IdbKeyRange\":[],\"IdbLocaleAwareKeyRange\":[\"IdbKeyRange\"],\"IdbMutableFile\":[\"EventTarget\"],\"IdbObjectStore\":[],\"IdbObjectStoreParameters\":[],\"IdbOpenDbOptions\":[],\"IdbOpenDbRequest\":[\"EventTarget\",\"IdbRequest\"],\"IdbRequest\":[\"EventTarget\"],\"IdbRequestReadyState\":[],\"IdbTransaction\":[\"EventTarget\"],\"IdbTransactionDurability\":[],\"IdbTransactionMode\":[],\"IdbTransactionOptions\":[],\"IdbVersionChangeEvent\":[\"Event\"],\"IdbVersionChangeEventInit\":[],\"IdleDeadline\":[],\"IdleRequestOptions\":[],\"IirFilterNode\":[\"AudioNode\",\"EventTarget\"],\"IirFilterOptions\":[],\"ImageBitmap\":[],\"ImageBitmapOptions\":[],\"ImageBitmapRenderingContext\":[],\"ImageCapture\":[],\"ImageCaptureError\":[],\"ImageCaptureErrorEvent\":[\"Event\"],\"ImageCaptureErrorEventInit\":[],\"ImageData\":[],\"ImageDecodeOptions\":[],\"ImageDecodeResult\":[],\"ImageDecoder\":[],\"ImageDecoderInit\":[],\"ImageEncodeOptions\":[],\"ImageOrientation\":[],\"ImageTrack\":[],\"ImageTrackList\":[],\"InputDeviceInfo\":[\"MediaDeviceInfo\"],\"InputEvent\":[\"Event\",\"UiEvent\"],\"InputEventInit\":[],\"IntersectionObserver\":[],\"IntersectionObserverEntry\":[],\"IntersectionObserverEntryInit\":[],\"IntersectionObserverInit\":[],\"IntlUtils\":[],\"IsInputPendingOptions\":[],\"IterableKeyAndValueResult\":[],\"IterableKeyOrValueResult\":[],\"IterationCompositeOperation\":[],\"JsonWebKey\":[],\"KeyAlgorithm\":[],\"KeyEvent\":[],\"KeyFrameRequestEvent\":[\"Event\"],\"KeyIdsInitData\":[],\"KeyboardEvent\":[\"Event\",\"UiEvent\"],\"KeyboardEventInit\":[],\"KeyframeAnimationOptions\":[],\"KeyframeEffect\":[\"AnimationEffect\"],\"KeyframeEffectOptions\":[],\"L10nElement\":[],\"L10nValue\":[],\"LargeBlobSupport\":[],\"LatencyMode\":[],\"LifecycleCallbacks\":[],\"LineAlignSetting\":[],\"ListBoxObject\":[],\"LocalMediaStream\":[\"EventTarget\",\"MediaStream\"],\"LocaleInfo\":[],\"Location\":[],\"Lock\":[],\"LockInfo\":[],\"LockManager\":[],\"LockManagerSnapshot\":[],\"LockMode\":[],\"LockOptions\":[],\"MathMlElement\":[\"Element\",\"EventTarget\",\"Node\"],\"MediaCapabilities\":[],\"MediaCapabilitiesInfo\":[],\"MediaConfiguration\":[],\"MediaDecodingConfiguration\":[],\"MediaDecodingType\":[],\"MediaDeviceInfo\":[],\"MediaDeviceKind\":[],\"MediaDevices\":[\"EventTarget\"],\"MediaElementAudioSourceNode\":[\"AudioNode\",\"EventTarget\"],\"MediaElementAudioSourceOptions\":[],\"MediaEncodingConfiguration\":[],\"MediaEncodingType\":[],\"MediaEncryptedEvent\":[\"Event\"],\"MediaError\":[],\"MediaImage\":[],\"MediaKeyError\":[\"Event\"],\"MediaKeyMessageEvent\":[\"Event\"],\"MediaKeyMessageEventInit\":[],\"MediaKeyMessageType\":[],\"MediaKeyNeededEventInit\":[],\"MediaKeySession\":[\"EventTarget\"],\"MediaKeySessionType\":[],\"MediaKeyStatus\":[],\"MediaKeyStatusMap\":[],\"MediaKeySystemAccess\":[],\"MediaKeySystemConfiguration\":[],\"MediaKeySystemMediaCapability\":[],\"MediaKeySystemStatus\":[],\"MediaKeys\":[],\"MediaKeysPolicy\":[],\"MediaKeysRequirement\":[],\"MediaList\":[],\"MediaMetadata\":[],\"MediaMetadataInit\":[],\"MediaPositionState\":[],\"MediaQueryList\":[\"EventTarget\"],\"MediaQueryListEvent\":[\"Event\"],\"MediaQueryListEventInit\":[],\"MediaRecorder\":[\"EventTarget\"],\"MediaRecorderErrorEvent\":[\"Event\"],\"MediaRecorderErrorEventInit\":[],\"MediaRecorderOptions\":[],\"MediaSession\":[],\"MediaSessionAction\":[],\"MediaSessionActionDetails\":[],\"MediaSessionPlaybackState\":[],\"MediaSettingsRange\":[],\"MediaSource\":[\"EventTarget\"],\"MediaSourceEndOfStreamError\":[],\"MediaSourceEnum\":[],\"MediaSourceReadyState\":[],\"MediaStream\":[\"EventTarget\"],\"MediaStreamAudioDestinationNode\":[\"AudioNode\",\"EventTarget\"],\"MediaStreamAudioSourceNode\":[\"AudioNode\",\"EventTarget\"],\"MediaStreamAudioSourceOptions\":[],\"MediaStreamConstraints\":[],\"MediaStreamError\":[],\"MediaStreamEvent\":[\"Event\"],\"MediaStreamEventInit\":[],\"MediaStreamTrack\":[\"EventTarget\"],\"MediaStreamTrackEvent\":[\"Event\"],\"MediaStreamTrackEventInit\":[],\"MediaStreamTrackGenerator\":[\"EventTarget\",\"MediaStreamTrack\"],\"MediaStreamTrackGeneratorInit\":[],\"MediaStreamTrackProcessor\":[],\"MediaStreamTrackProcessorInit\":[],\"MediaStreamTrackState\":[],\"MediaTrackCapabilities\":[],\"MediaTrackConstraintSet\":[],\"MediaTrackConstraints\":[],\"MediaTrackSettings\":[],\"MediaTrackSupportedConstraints\":[],\"MemoryAttribution\":[],\"MemoryAttributionContainer\":[],\"MemoryBreakdownEntry\":[],\"MemoryMeasurement\":[],\"MessageChannel\":[],\"MessageEvent\":[\"Event\"],\"MessageEventInit\":[],\"MessagePort\":[\"EventTarget\"],\"MeteringMode\":[],\"MidiAccess\":[\"EventTarget\"],\"MidiConnectionEvent\":[\"Event\"],\"MidiConnectionEventInit\":[],\"MidiInput\":[\"EventTarget\",\"MidiPort\"],\"MidiInputMap\":[],\"MidiMessageEvent\":[\"Event\"],\"MidiMessageEventInit\":[],\"MidiOptions\":[],\"MidiOutput\":[\"EventTarget\",\"MidiPort\"],\"MidiOutputMap\":[],\"MidiPort\":[\"EventTarget\"],\"MidiPortConnectionState\":[],\"MidiPortDeviceState\":[],\"MidiPortType\":[],\"MimeType\":[],\"MimeTypeArray\":[],\"MouseEvent\":[\"Event\",\"UiEvent\"],\"MouseEventInit\":[],\"MouseScrollEvent\":[\"Event\",\"MouseEvent\",\"UiEvent\"],\"MozDebug\":[],\"MutationEvent\":[\"Event\"],\"MutationObserver\":[],\"MutationObserverInit\":[],\"MutationObservingInfo\":[],\"MutationRecord\":[],\"NamedNodeMap\":[],\"NativeOsFileReadOptions\":[],\"NativeOsFileWriteAtomicOptions\":[],\"NavigationType\":[],\"Navigator\":[],\"NavigatorAutomationInformation\":[],\"NavigatorUaBrandVersion\":[],\"NavigatorUaData\":[],\"NetworkCommandOptions\":[],\"NetworkInformation\":[\"EventTarget\"],\"NetworkResultOptions\":[],\"Node\":[\"EventTarget\"],\"NodeFilter\":[],\"NodeIterator\":[],\"NodeList\":[],\"Notification\":[\"EventTarget\"],\"NotificationAction\":[],\"NotificationDirection\":[],\"NotificationEvent\":[\"Event\",\"ExtendableEvent\"],\"NotificationEventInit\":[],\"NotificationOptions\":[],\"NotificationPermission\":[],\"ObserverCallback\":[],\"OesElementIndexUint\":[],\"OesStandardDerivatives\":[],\"OesTextureFloat\":[],\"OesTextureFloatLinear\":[],\"OesTextureHalfFloat\":[],\"OesTextureHalfFloatLinear\":[],\"OesVertexArrayObject\":[],\"OfflineAudioCompletionEvent\":[\"Event\"],\"OfflineAudioCompletionEventInit\":[],\"OfflineAudioContext\":[\"BaseAudioContext\",\"EventTarget\"],\"OfflineAudioContextOptions\":[],\"OfflineResourceList\":[\"EventTarget\"],\"OffscreenCanvas\":[\"EventTarget\"],\"OffscreenCanvasRenderingContext2d\":[],\"OpenFilePickerOptions\":[],\"OpenWindowEventDetail\":[],\"OptionalEffectTiming\":[],\"OrientationLockType\":[],\"OrientationType\":[],\"OscillatorNode\":[\"AudioNode\",\"AudioScheduledSourceNode\",\"EventTarget\"],\"OscillatorOptions\":[],\"OscillatorType\":[],\"OverSampleType\":[],\"OvrMultiview2\":[],\"PageTransitionEvent\":[\"Event\"],\"PageTransitionEventInit\":[],\"PaintRequest\":[],\"PaintRequestList\":[],\"PaintWorkletGlobalScope\":[\"WorkletGlobalScope\"],\"PannerNode\":[\"AudioNode\",\"EventTarget\"],\"PannerOptions\":[],\"PanningModelType\":[],\"ParityType\":[],\"Path2d\":[],\"PaymentAddress\":[],\"PaymentComplete\":[],\"PaymentMethodChangeEvent\":[\"Event\",\"PaymentRequestUpdateEvent\"],\"PaymentMethodChangeEventInit\":[],\"PaymentRequestUpdateEvent\":[\"Event\"],\"PaymentRequestUpdateEventInit\":[],\"PaymentResponse\":[],\"Pbkdf2Params\":[],\"PcImplIceConnectionState\":[],\"PcImplIceGatheringState\":[],\"PcImplSignalingState\":[],\"PcObserverStateType\":[],\"Performance\":[\"EventTarget\"],\"PerformanceEntry\":[],\"PerformanceEntryEventInit\":[],\"PerformanceEntryFilterOptions\":[],\"PerformanceMark\":[\"PerformanceEntry\"],\"PerformanceMarkOptions\":[],\"PerformanceMeasure\":[\"PerformanceEntry\"],\"PerformanceMeasureOptions\":[],\"PerformanceNavigation\":[],\"PerformanceNavigationTiming\":[\"PerformanceEntry\",\"PerformanceResourceTiming\"],\"PerformanceObserver\":[],\"PerformanceObserverEntryList\":[],\"PerformanceObserverInit\":[],\"PerformanceResourceTiming\":[\"PerformanceEntry\"],\"PerformanceServerTiming\":[],\"PerformanceTiming\":[],\"PeriodicWave\":[],\"PeriodicWaveConstraints\":[],\"PeriodicWaveOptions\":[],\"PermissionDescriptor\":[],\"PermissionName\":[],\"PermissionState\":[],\"PermissionStatus\":[\"EventTarget\"],\"Permissions\":[],\"PhotoCapabilities\":[],\"PhotoSettings\":[],\"PictureInPictureEvent\":[\"Event\"],\"PictureInPictureEventInit\":[],\"PictureInPictureWindow\":[\"EventTarget\"],\"PlaneLayout\":[],\"PlaybackDirection\":[],\"Plugin\":[],\"PluginArray\":[],\"PluginCrashedEventInit\":[],\"Point2d\":[],\"PointerEvent\":[\"Event\",\"MouseEvent\",\"UiEvent\"],\"PointerEventInit\":[],\"PopStateEvent\":[\"Event\"],\"PopStateEventInit\":[],\"PopupBlockedEvent\":[\"Event\"],\"PopupBlockedEventInit\":[],\"Position\":[],\"PositionAlignSetting\":[],\"PositionError\":[],\"PositionOptions\":[],\"PremultiplyAlpha\":[],\"Presentation\":[],\"PresentationAvailability\":[\"EventTarget\"],\"PresentationConnection\":[\"EventTarget\"],\"PresentationConnectionAvailableEvent\":[\"Event\"],\"PresentationConnectionAvailableEventInit\":[],\"PresentationConnectionBinaryType\":[],\"PresentationConnectionCloseEvent\":[\"Event\"],\"PresentationConnectionCloseEventInit\":[],\"PresentationConnectionClosedReason\":[],\"PresentationConnectionList\":[\"EventTarget\"],\"PresentationConnectionState\":[],\"PresentationReceiver\":[],\"PresentationRequest\":[\"EventTarget\"],\"PresentationStyle\":[],\"ProcessingInstruction\":[\"CharacterData\",\"EventTarget\",\"Node\"],\"ProfileTimelineLayerRect\":[],\"ProfileTimelineMarker\":[],\"ProfileTimelineMessagePortOperationType\":[],\"ProfileTimelineStackFrame\":[],\"ProfileTimelineWorkerOperationType\":[],\"ProgressEvent\":[\"Event\"],\"ProgressEventInit\":[],\"PromiseNativeHandler\":[],\"PromiseRejectionEvent\":[\"Event\"],\"PromiseRejectionEventInit\":[],\"PublicKeyCredential\":[\"Credential\"],\"PublicKeyCredentialCreationOptions\":[],\"PublicKeyCredentialCreationOptionsJson\":[],\"PublicKeyCredentialDescriptor\":[],\"PublicKeyCredentialDescriptorJson\":[],\"PublicKeyCredentialEntity\":[],\"PublicKeyCredentialHints\":[],\"PublicKeyCredentialParameters\":[],\"PublicKeyCredentialRequestOptions\":[],\"PublicKeyCredentialRequestOptionsJson\":[],\"PublicKeyCredentialRpEntity\":[],\"PublicKeyCredentialType\":[],\"PublicKeyCredentialUserEntity\":[],\"PublicKeyCredentialUserEntityJson\":[],\"PushEncryptionKeyName\":[],\"PushEvent\":[\"Event\",\"ExtendableEvent\"],\"PushEventInit\":[],\"PushManager\":[],\"PushMessageData\":[],\"PushPermissionState\":[],\"PushSubscription\":[],\"PushSubscriptionInit\":[],\"PushSubscriptionJson\":[],\"PushSubscriptionKeys\":[],\"PushSubscriptionOptions\":[],\"PushSubscriptionOptionsInit\":[],\"QueryOptions\":[],\"QueuingStrategy\":[],\"QueuingStrategyInit\":[],\"RadioNodeList\":[\"NodeList\"],\"Range\":[\"AbstractRange\"],\"RcwnPerfStats\":[],\"RcwnStatus\":[],\"ReadableByteStreamController\":[],\"ReadableStream\":[],\"ReadableStreamByobReader\":[],\"ReadableStreamByobRequest\":[],\"ReadableStreamDefaultController\":[],\"ReadableStreamDefaultReader\":[],\"ReadableStreamGetReaderOptions\":[],\"ReadableStreamIteratorOptions\":[],\"ReadableStreamReadResult\":[],\"ReadableStreamReaderMode\":[],\"ReadableStreamType\":[],\"ReadableWritablePair\":[],\"RecordingState\":[],\"RedEyeReduction\":[],\"ReferrerPolicy\":[],\"RegisterRequest\":[],\"RegisterResponse\":[],\"RegisteredKey\":[],\"RegistrationOptions\":[],\"RegistrationResponseJson\":[],\"Request\":[],\"RequestCache\":[],\"RequestCredentials\":[],\"RequestDestination\":[],\"RequestDeviceOptions\":[],\"RequestInit\":[],\"RequestMediaKeySystemAccessNotification\":[],\"RequestMode\":[],\"RequestRedirect\":[],\"ResidentKeyRequirement\":[],\"ResizeObserver\":[],\"ResizeObserverBoxOptions\":[],\"ResizeObserverEntry\":[],\"ResizeObserverOptions\":[],\"ResizeObserverSize\":[],\"ResizeQuality\":[],\"Response\":[],\"ResponseInit\":[],\"ResponseType\":[],\"RsaHashedImportParams\":[],\"RsaOaepParams\":[],\"RsaOtherPrimesInfo\":[],\"RsaPssParams\":[],\"RtcAnswerOptions\":[],\"RtcBundlePolicy\":[],\"RtcCertificate\":[],\"RtcCertificateExpiration\":[],\"RtcCodecStats\":[],\"RtcConfiguration\":[],\"RtcDataChannel\":[\"EventTarget\"],\"RtcDataChannelEvent\":[\"Event\"],\"RtcDataChannelEventInit\":[],\"RtcDataChannelInit\":[],\"RtcDataChannelState\":[],\"RtcDataChannelType\":[],\"RtcDegradationPreference\":[],\"RtcEncodedAudioFrame\":[],\"RtcEncodedAudioFrameMetadata\":[],\"RtcEncodedAudioFrameOptions\":[],\"RtcEncodedVideoFrame\":[],\"RtcEncodedVideoFrameMetadata\":[],\"RtcEncodedVideoFrameOptions\":[],\"RtcEncodedVideoFrameType\":[],\"RtcFecParameters\":[],\"RtcIceCandidate\":[],\"RtcIceCandidateInit\":[],\"RtcIceCandidatePairStats\":[],\"RtcIceCandidateStats\":[],\"RtcIceComponentStats\":[],\"RtcIceConnectionState\":[],\"RtcIceCredentialType\":[],\"RtcIceGatheringState\":[],\"RtcIceServer\":[],\"RtcIceTransportPolicy\":[],\"RtcIdentityAssertion\":[],\"RtcIdentityAssertionResult\":[],\"RtcIdentityProvider\":[],\"RtcIdentityProviderDetails\":[],\"RtcIdentityProviderOptions\":[],\"RtcIdentityProviderRegistrar\":[],\"RtcIdentityValidationResult\":[],\"RtcInboundRtpStreamStats\":[],\"RtcMediaStreamStats\":[],\"RtcMediaStreamTrackStats\":[],\"RtcOfferAnswerOptions\":[],\"RtcOfferOptions\":[],\"RtcOutboundRtpStreamStats\":[],\"RtcPeerConnection\":[\"EventTarget\"],\"RtcPeerConnectionIceErrorEvent\":[\"Event\"],\"RtcPeerConnectionIceEvent\":[\"Event\"],\"RtcPeerConnectionIceEventInit\":[],\"RtcPeerConnectionState\":[],\"RtcPriorityType\":[],\"RtcRtcpParameters\":[],\"RtcRtpCapabilities\":[],\"RtcRtpCodecCapability\":[],\"RtcRtpCodecParameters\":[],\"RtcRtpContributingSource\":[],\"RtcRtpEncodingParameters\":[],\"RtcRtpHeaderExtensionCapability\":[],\"RtcRtpHeaderExtensionParameters\":[],\"RtcRtpParameters\":[],\"RtcRtpReceiver\":[],\"RtcRtpScriptTransform\":[],\"RtcRtpScriptTransformer\":[\"EventTarget\"],\"RtcRtpSender\":[],\"RtcRtpSourceEntry\":[],\"RtcRtpSourceEntryType\":[],\"RtcRtpSynchronizationSource\":[],\"RtcRtpTransceiver\":[],\"RtcRtpTransceiverDirection\":[],\"RtcRtpTransceiverInit\":[],\"RtcRtxParameters\":[],\"RtcSdpType\":[],\"RtcSessionDescription\":[],\"RtcSessionDescriptionInit\":[],\"RtcSignalingState\":[],\"RtcStats\":[],\"RtcStatsIceCandidatePairState\":[],\"RtcStatsIceCandidateType\":[],\"RtcStatsReport\":[],\"RtcStatsReportInternal\":[],\"RtcStatsType\":[],\"RtcTrackEvent\":[\"Event\"],\"RtcTrackEventInit\":[],\"RtcTransformEvent\":[\"Event\"],\"RtcTransportStats\":[],\"RtcdtmfSender\":[\"EventTarget\"],\"RtcdtmfToneChangeEvent\":[\"Event\"],\"RtcdtmfToneChangeEventInit\":[],\"RtcrtpContributingSourceStats\":[],\"RtcrtpStreamStats\":[],\"SFrameTransform\":[\"EventTarget\"],\"SFrameTransformErrorEvent\":[\"Event\"],\"SFrameTransformErrorEventInit\":[],\"SFrameTransformErrorEventType\":[],\"SFrameTransformOptions\":[],\"SFrameTransformRole\":[],\"SaveFilePickerOptions\":[],\"Scheduler\":[],\"SchedulerPostTaskOptions\":[],\"Scheduling\":[],\"Screen\":[\"EventTarget\"],\"ScreenColorGamut\":[],\"ScreenDetailed\":[\"EventTarget\",\"Screen\"],\"ScreenDetails\":[\"EventTarget\"],\"ScreenLuminance\":[],\"ScreenOrientation\":[\"EventTarget\"],\"ScriptProcessorNode\":[\"AudioNode\",\"EventTarget\"],\"ScrollAreaEvent\":[\"Event\",\"UiEvent\"],\"ScrollBehavior\":[],\"ScrollBoxObject\":[],\"ScrollIntoViewContainer\":[],\"ScrollIntoViewOptions\":[],\"ScrollLogicalPosition\":[],\"ScrollOptions\":[],\"ScrollRestoration\":[],\"ScrollSetting\":[],\"ScrollState\":[],\"ScrollToOptions\":[],\"ScrollViewChangeEventInit\":[],\"SecurityPolicyViolationEvent\":[\"Event\"],\"SecurityPolicyViolationEventDisposition\":[],\"SecurityPolicyViolationEventInit\":[],\"Selection\":[],\"SelectionMode\":[],\"Serial\":[\"EventTarget\"],\"SerialInputSignals\":[],\"SerialOptions\":[],\"SerialOutputSignals\":[],\"SerialPort\":[\"EventTarget\"],\"SerialPortFilter\":[],\"SerialPortInfo\":[],\"SerialPortRequestOptions\":[],\"ServerSocketOptions\":[],\"ServiceWorker\":[\"EventTarget\"],\"ServiceWorkerContainer\":[\"EventTarget\"],\"ServiceWorkerGlobalScope\":[\"EventTarget\",\"WorkerGlobalScope\"],\"ServiceWorkerRegistration\":[\"EventTarget\"],\"ServiceWorkerState\":[],\"ServiceWorkerUpdateViaCache\":[],\"ShadowRoot\":[\"DocumentFragment\",\"EventTarget\",\"Node\"],\"ShadowRootInit\":[],\"ShadowRootMode\":[],\"ShareData\":[],\"SharedWorker\":[\"EventTarget\"],\"SharedWorkerGlobalScope\":[\"EventTarget\",\"WorkerGlobalScope\"],\"ShowPopoverOptions\":[],\"SignResponse\":[],\"SocketElement\":[],\"SocketOptions\":[],\"SocketReadyState\":[],\"SocketsDict\":[],\"SourceBuffer\":[\"EventTarget\"],\"SourceBufferAppendMode\":[],\"SourceBufferList\":[\"EventTarget\"],\"SpeechGrammar\":[],\"SpeechGrammarList\":[],\"SpeechRecognition\":[\"EventTarget\"],\"SpeechRecognitionAlternative\":[],\"SpeechRecognitionError\":[\"Event\"],\"SpeechRecognitionErrorCode\":[],\"SpeechRecognitionErrorInit\":[],\"SpeechRecognitionEvent\":[\"Event\"],\"SpeechRecognitionEventInit\":[],\"SpeechRecognitionResult\":[],\"SpeechRecognitionResultList\":[],\"SpeechSynthesis\":[\"EventTarget\"],\"SpeechSynthesisErrorCode\":[],\"SpeechSynthesisErrorEvent\":[\"Event\",\"SpeechSynthesisEvent\"],\"SpeechSynthesisErrorEventInit\":[],\"SpeechSynthesisEvent\":[\"Event\"],\"SpeechSynthesisEventInit\":[],\"SpeechSynthesisUtterance\":[\"EventTarget\"],\"SpeechSynthesisVoice\":[],\"StartViewTransitionOptions\":[],\"StaticRange\":[\"AbstractRange\"],\"StaticRangeInit\":[],\"StereoPannerNode\":[\"AudioNode\",\"EventTarget\"],\"StereoPannerOptions\":[],\"Storage\":[],\"StorageEstimate\":[],\"StorageEvent\":[\"Event\"],\"StorageEventInit\":[],\"StorageManager\":[],\"StorageType\":[],\"StreamPipeOptions\":[],\"StyleRuleChangeEventInit\":[],\"StyleSheet\":[],\"StyleSheetApplicableStateChangeEventInit\":[],\"StyleSheetChangeEventInit\":[],\"StyleSheetList\":[],\"SubmitEvent\":[\"Event\"],\"SubmitEventInit\":[],\"SubtleCrypto\":[],\"SupportedType\":[],\"SvcOutputMetadata\":[],\"SvgAngle\":[],\"SvgAnimateElement\":[\"Element\",\"EventTarget\",\"Node\",\"SvgAnimationElement\",\"SvgElement\"],\"SvgAnimateMotionElement\":[\"Element\",\"EventTarget\",\"Node\",\"SvgAnimationElement\",\"SvgElement\"],\"SvgAnimateTransformElement\":[\"Element\",\"EventTarget\",\"Node\",\"SvgAnimationElement\",\"SvgElement\"],\"SvgAnimatedAngle\":[],\"SvgAnimatedBoolean\":[],\"SvgAnimatedEnumeration\":[],\"SvgAnimatedInteger\":[],\"SvgAnimatedLength\":[],\"SvgAnimatedLengthList\":[],\"SvgAnimatedNumber\":[],\"SvgAnimatedNumberList\":[],\"SvgAnimatedPreserveAspectRatio\":[],\"SvgAnimatedRect\":[],\"SvgAnimatedString\":[],\"SvgAnimatedTransformList\":[],\"SvgAnimationElement\":[\"Element\",\"EventTarget\",\"Node\",\"SvgElement\"],\"SvgBoundingBoxOptions\":[],\"SvgCircleElement\":[\"Element\",\"EventTarget\",\"Node\",\"SvgElement\",\"SvgGeometryElement\",\"SvgGraphicsElement\"],\"SvgClipPathElement\":[\"Element\",\"EventTarget\",\"Node\",\"SvgElement\"],\"SvgComponentTransferFunctionElement\":[\"Element\",\"EventTarget\",\"Node\",\"SvgElement\"],\"SvgDefsElement\":[\"Element\",\"EventTarget\",\"Node\",\"SvgElement\",\"SvgGraphicsElement\"],\"SvgDescElement\":[\"Element\",\"EventTarget\",\"Node\",\"SvgElement\"],\"SvgElement\":[\"Element\",\"EventTarget\",\"Node\"],\"SvgEllipseElement\":[\"Element\",\"EventTarget\",\"Node\",\"SvgElement\",\"SvgGeometryElement\",\"SvgGraphicsElement\"],\"SvgFilterElement\":[\"Element\",\"EventTarget\",\"Node\",\"SvgElement\"],\"SvgForeignObjectElement\":[\"Element\",\"EventTarget\",\"Node\",\"SvgElement\",\"SvgGraphicsElement\"],\"SvgGeometryElement\":[\"Element\",\"EventTarget\",\"Node\",\"SvgElement\",\"SvgGraphicsElement\"],\"SvgGradientElement\":[\"Element\",\"EventTarget\",\"Node\",\"SvgElement\"],\"SvgGraphicsElement\":[\"Element\",\"EventTarget\",\"Node\",\"SvgElement\"],\"SvgImageElement\":[\"Element\",\"EventTarget\",\"Node\",\"SvgElement\",\"SvgGraphicsElement\"],\"SvgLength\":[],\"SvgLengthList\":[],\"SvgLineElement\":[\"Element\",\"EventTarget\",\"Node\",\"SvgElement\",\"SvgGeometryElement\",\"SvgGraphicsElement\"],\"SvgLinearGradientElement\":[\"Element\",\"EventTarget\",\"Node\",\"SvgElement\",\"SvgGradientElement\"],\"SvgMarkerElement\":[\"Element\",\"EventTarget\",\"Node\",\"SvgElement\"],\"SvgMaskElement\":[\"Element\",\"EventTarget\",\"Node\",\"SvgElement\"],\"SvgMatrix\":[],\"SvgMetadataElement\":[\"Element\",\"EventTarget\",\"Node\",\"SvgElement\"],\"SvgNumber\":[],\"SvgNumberList\":[],\"SvgPathElement\":[\"Element\",\"EventTarget\",\"Node\",\"SvgElement\",\"SvgGeometryElement\",\"SvgGraphicsElement\"],\"SvgPathSeg\":[],\"SvgPathSegArcAbs\":[\"SvgPathSeg\"],\"SvgPathSegArcRel\":[\"SvgPathSeg\"],\"SvgPathSegClosePath\":[\"SvgPathSeg\"],\"SvgPathSegCurvetoCubicAbs\":[\"SvgPathSeg\"],\"SvgPathSegCurvetoCubicRel\":[\"SvgPathSeg\"],\"SvgPathSegCurvetoCubicSmoothAbs\":[\"SvgPathSeg\"],\"SvgPathSegCurvetoCubicSmoothRel\":[\"SvgPathSeg\"],\"SvgPathSegCurvetoQuadraticAbs\":[\"SvgPathSeg\"],\"SvgPathSegCurvetoQuadraticRel\":[\"SvgPathSeg\"],\"SvgPathSegCurvetoQuadraticSmoothAbs\":[\"SvgPathSeg\"],\"SvgPathSegCurvetoQuadraticSmoothRel\":[\"SvgPathSeg\"],\"SvgPathSegLinetoAbs\":[\"SvgPathSeg\"],\"SvgPathSegLinetoHorizontalAbs\":[\"SvgPathSeg\"],\"SvgPathSegLinetoHorizontalRel\":[\"SvgPathSeg\"],\"SvgPathSegLinetoRel\":[\"SvgPathSeg\"],\"SvgPathSegLinetoVerticalAbs\":[\"SvgPathSeg\"],\"SvgPathSegLinetoVerticalRel\":[\"SvgPathSeg\"],\"SvgPathSegList\":[],\"SvgPathSegMovetoAbs\":[\"SvgPathSeg\"],\"SvgPathSegMovetoRel\":[\"SvgPathSeg\"],\"SvgPatternElement\":[\"Element\",\"EventTarget\",\"Node\",\"SvgElement\"],\"SvgPoint\":[],\"SvgPointList\":[],\"SvgPolygonElement\":[\"Element\",\"EventTarget\",\"Node\",\"SvgElement\",\"SvgGeometryElement\",\"SvgGraphicsElement\"],\"SvgPolylineElement\":[\"Element\",\"EventTarget\",\"Node\",\"SvgElement\",\"SvgGeometryElement\",\"SvgGraphicsElement\"],\"SvgPreserveAspectRatio\":[],\"SvgRadialGradientElement\":[\"Element\",\"EventTarget\",\"Node\",\"SvgElement\",\"SvgGradientElement\"],\"SvgRect\":[],\"SvgRectElement\":[\"Element\",\"EventTarget\",\"Node\",\"SvgElement\",\"SvgGeometryElement\",\"SvgGraphicsElement\"],\"SvgScriptElement\":[\"Element\",\"EventTarget\",\"Node\",\"SvgElement\"],\"SvgSetElement\":[\"Element\",\"EventTarget\",\"Node\",\"SvgAnimationElement\",\"SvgElement\"],\"SvgStopElement\":[\"Element\",\"EventTarget\",\"Node\",\"SvgElement\"],\"SvgStringList\":[],\"SvgStyleElement\":[\"Element\",\"EventTarget\",\"Node\",\"SvgElement\"],\"SvgSwitchElement\":[\"Element\",\"EventTarget\",\"Node\",\"SvgElement\",\"SvgGraphicsElement\"],\"SvgSymbolElement\":[\"Element\",\"EventTarget\",\"Node\",\"SvgElement\"],\"SvgTextContentElement\":[\"Element\",\"EventTarget\",\"Node\",\"SvgElement\",\"SvgGraphicsElement\"],\"SvgTextElement\":[\"Element\",\"EventTarget\",\"Node\",\"SvgElement\",\"SvgGraphicsElement\",\"SvgTextContentElement\",\"SvgTextPositioningElement\"],\"SvgTextPathElement\":[\"Element\",\"EventTarget\",\"Node\",\"SvgElement\",\"SvgGraphicsElement\",\"SvgTextContentElement\"],\"SvgTextPositioningElement\":[\"Element\",\"EventTarget\",\"Node\",\"SvgElement\",\"SvgGraphicsElement\",\"SvgTextContentElement\"],\"SvgTitleElement\":[\"Element\",\"EventTarget\",\"Node\",\"SvgElement\"],\"SvgTransform\":[],\"SvgTransformList\":[],\"SvgUnitTypes\":[],\"SvgUseElement\":[\"Element\",\"EventTarget\",\"Node\",\"SvgElement\",\"SvgGraphicsElement\"],\"SvgViewElement\":[\"Element\",\"EventTarget\",\"Node\",\"SvgElement\"],\"SvgZoomAndPan\":[],\"SvgaElement\":[\"Element\",\"EventTarget\",\"Node\",\"SvgElement\",\"SvgGraphicsElement\"],\"SvgfeBlendElement\":[\"Element\",\"EventTarget\",\"Node\",\"SvgElement\"],\"SvgfeColorMatrixElement\":[\"Element\",\"EventTarget\",\"Node\",\"SvgElement\"],\"SvgfeComponentTransferElement\":[\"Element\",\"EventTarget\",\"Node\",\"SvgElement\"],\"SvgfeCompositeElement\":[\"Element\",\"EventTarget\",\"Node\",\"SvgElement\"],\"SvgfeConvolveMatrixElement\":[\"Element\",\"EventTarget\",\"Node\",\"SvgElement\"],\"SvgfeDiffuseLightingElement\":[\"Element\",\"EventTarget\",\"Node\",\"SvgElement\"],\"SvgfeDisplacementMapElement\":[\"Element\",\"EventTarget\",\"Node\",\"SvgElement\"],\"SvgfeDistantLightElement\":[\"Element\",\"EventTarget\",\"Node\",\"SvgElement\"],\"SvgfeDropShadowElement\":[\"Element\",\"EventTarget\",\"Node\",\"SvgElement\"],\"SvgfeFloodElement\":[\"Element\",\"EventTarget\",\"Node\",\"SvgElement\"],\"SvgfeFuncAElement\":[\"Element\",\"EventTarget\",\"Node\",\"SvgComponentTransferFunctionElement\",\"SvgElement\"],\"SvgfeFuncBElement\":[\"Element\",\"EventTarget\",\"Node\",\"SvgComponentTransferFunctionElement\",\"SvgElement\"],\"SvgfeFuncGElement\":[\"Element\",\"EventTarget\",\"Node\",\"SvgComponentTransferFunctionElement\",\"SvgElement\"],\"SvgfeFuncRElement\":[\"Element\",\"EventTarget\",\"Node\",\"SvgComponentTransferFunctionElement\",\"SvgElement\"],\"SvgfeGaussianBlurElement\":[\"Element\",\"EventTarget\",\"Node\",\"SvgElement\"],\"SvgfeImageElement\":[\"Element\",\"EventTarget\",\"Node\",\"SvgElement\"],\"SvgfeMergeElement\":[\"Element\",\"EventTarget\",\"Node\",\"SvgElement\"],\"SvgfeMergeNodeElement\":[\"Element\",\"EventTarget\",\"Node\",\"SvgElement\"],\"SvgfeMorphologyElement\":[\"Element\",\"EventTarget\",\"Node\",\"SvgElement\"],\"SvgfeOffsetElement\":[\"Element\",\"EventTarget\",\"Node\",\"SvgElement\"],\"SvgfePointLightElement\":[\"Element\",\"EventTarget\",\"Node\",\"SvgElement\"],\"SvgfeSpecularLightingElement\":[\"Element\",\"EventTarget\",\"Node\",\"SvgElement\"],\"SvgfeSpotLightElement\":[\"Element\",\"EventTarget\",\"Node\",\"SvgElement\"],\"SvgfeTileElement\":[\"Element\",\"EventTarget\",\"Node\",\"SvgElement\"],\"SvgfeTurbulenceElement\":[\"Element\",\"EventTarget\",\"Node\",\"SvgElement\"],\"SvggElement\":[\"Element\",\"EventTarget\",\"Node\",\"SvgElement\",\"SvgGraphicsElement\"],\"SvgmPathElement\":[\"Element\",\"EventTarget\",\"Node\",\"SvgElement\"],\"SvgsvgElement\":[\"Element\",\"EventTarget\",\"Node\",\"SvgElement\",\"SvgGraphicsElement\"],\"SvgtSpanElement\":[\"Element\",\"EventTarget\",\"Node\",\"SvgElement\",\"SvgGraphicsElement\",\"SvgTextContentElement\",\"SvgTextPositioningElement\"],\"TaskController\":[\"AbortController\"],\"TaskControllerInit\":[],\"TaskPriority\":[],\"TaskPriorityChangeEvent\":[\"Event\"],\"TaskPriorityChangeEventInit\":[],\"TaskSignal\":[\"AbortSignal\",\"EventTarget\"],\"TaskSignalAnyInit\":[],\"TcpReadyState\":[],\"TcpServerSocket\":[\"EventTarget\"],\"TcpServerSocketEvent\":[\"Event\"],\"TcpServerSocketEventInit\":[],\"TcpSocket\":[\"EventTarget\"],\"TcpSocketBinaryType\":[],\"TcpSocketErrorEvent\":[\"Event\"],\"TcpSocketErrorEventInit\":[],\"TcpSocketEvent\":[\"Event\"],\"TcpSocketEventInit\":[],\"Text\":[\"CharacterData\",\"EventTarget\",\"Node\"],\"TextDecodeOptions\":[],\"TextDecoder\":[],\"TextDecoderOptions\":[],\"TextEncoder\":[],\"TextMetrics\":[],\"TextTrack\":[\"EventTarget\"],\"TextTrackCue\":[\"EventTarget\"],\"TextTrackCueList\":[],\"TextTrackKind\":[],\"TextTrackList\":[\"EventTarget\"],\"TextTrackMode\":[],\"TimeEvent\":[\"Event\"],\"TimeRanges\":[],\"ToggleEvent\":[\"Event\"],\"ToggleEventInit\":[],\"TogglePopoverOptions\":[],\"TokenBinding\":[],\"TokenBindingStatus\":[],\"Touch\":[],\"TouchEvent\":[\"Event\",\"UiEvent\"],\"TouchEventInit\":[],\"TouchInit\":[],\"TouchList\":[],\"TrackEvent\":[\"Event\"],\"TrackEventInit\":[],\"TransformStream\":[],\"TransformStreamDefaultController\":[],\"Transformer\":[],\"TransitionEvent\":[\"Event\"],\"TransitionEventInit\":[],\"Transport\":[],\"TreeBoxObject\":[],\"TreeCellInfo\":[],\"TreeView\":[],\"TreeWalker\":[],\"U2f\":[],\"U2fClientData\":[],\"ULongRange\":[],\"UaDataValues\":[],\"UaLowEntropyJson\":[],\"UdpMessageEventInit\":[],\"UdpOptions\":[],\"UiEvent\":[\"Event\"],\"UiEventInit\":[],\"UnderlyingSink\":[],\"UnderlyingSource\":[],\"Url\":[],\"UrlSearchParams\":[],\"Usb\":[\"EventTarget\"],\"UsbAlternateInterface\":[],\"UsbConfiguration\":[],\"UsbConnectionEvent\":[\"Event\"],\"UsbConnectionEventInit\":[],\"UsbControlTransferParameters\":[],\"UsbDevice\":[],\"UsbDeviceFilter\":[],\"UsbDeviceRequestOptions\":[],\"UsbDirection\":[],\"UsbEndpoint\":[],\"UsbEndpointType\":[],\"UsbInTransferResult\":[],\"UsbInterface\":[],\"UsbIsochronousInTransferPacket\":[],\"UsbIsochronousInTransferResult\":[],\"UsbIsochronousOutTransferPacket\":[],\"UsbIsochronousOutTransferResult\":[],\"UsbOutTransferResult\":[],\"UsbPermissionDescriptor\":[],\"UsbPermissionResult\":[\"EventTarget\",\"PermissionStatus\"],\"UsbPermissionStorage\":[],\"UsbRecipient\":[],\"UsbRequestType\":[],\"UsbTransferStatus\":[],\"UserActivation\":[],\"UserProximityEvent\":[\"Event\"],\"UserProximityEventInit\":[],\"UserVerificationRequirement\":[],\"ValidityState\":[],\"ValueEvent\":[\"Event\"],\"ValueEventInit\":[],\"VideoColorPrimaries\":[],\"VideoColorSpace\":[],\"VideoColorSpaceInit\":[],\"VideoConfiguration\":[],\"VideoDecoder\":[\"EventTarget\"],\"VideoDecoderConfig\":[],\"VideoDecoderInit\":[],\"VideoDecoderSupport\":[],\"VideoEncoder\":[\"EventTarget\"],\"VideoEncoderBitrateMode\":[],\"VideoEncoderConfig\":[],\"VideoEncoderEncodeOptions\":[],\"VideoEncoderInit\":[],\"VideoEncoderSupport\":[],\"VideoFacingModeEnum\":[],\"VideoFrame\":[],\"VideoFrameBufferInit\":[],\"VideoFrameCopyToOptions\":[],\"VideoFrameInit\":[],\"VideoFrameMetadata\":[],\"VideoMatrixCoefficients\":[],\"VideoPixelFormat\":[],\"VideoPlaybackQuality\":[],\"VideoStreamTrack\":[\"EventTarget\",\"MediaStreamTrack\"],\"VideoTrack\":[],\"VideoTrackList\":[\"EventTarget\"],\"VideoTransferCharacteristics\":[],\"ViewTransition\":[],\"ViewTransitionTypeSet\":[],\"VisibilityState\":[],\"VisualViewport\":[\"EventTarget\"],\"VoidCallback\":[],\"VrDisplay\":[\"EventTarget\"],\"VrDisplayCapabilities\":[],\"VrEye\":[],\"VrEyeParameters\":[],\"VrFieldOfView\":[],\"VrFrameData\":[],\"VrLayer\":[],\"VrMockController\":[],\"VrMockDisplay\":[],\"VrPose\":[],\"VrServiceTest\":[],\"VrStageParameters\":[],\"VrSubmitFrameResult\":[],\"VttCue\":[\"EventTarget\",\"TextTrackCue\"],\"VttRegion\":[],\"WakeLock\":[],\"WakeLockSentinel\":[\"EventTarget\"],\"WakeLockType\":[],\"WatchAdvertisementsOptions\":[],\"WaveShaperNode\":[\"AudioNode\",\"EventTarget\"],\"WaveShaperOptions\":[],\"WebGl2RenderingContext\":[],\"WebGlActiveInfo\":[],\"WebGlBuffer\":[],\"WebGlContextAttributes\":[],\"WebGlContextEvent\":[\"Event\"],\"WebGlContextEventInit\":[],\"WebGlFramebuffer\":[],\"WebGlPowerPreference\":[],\"WebGlProgram\":[],\"WebGlQuery\":[],\"WebGlRenderbuffer\":[],\"WebGlRenderingContext\":[],\"WebGlSampler\":[],\"WebGlShader\":[],\"WebGlShaderPrecisionFormat\":[],\"WebGlSync\":[],\"WebGlTexture\":[],\"WebGlTransformFeedback\":[],\"WebGlUniformLocation\":[],\"WebGlVertexArrayObject\":[],\"WebKitCssMatrix\":[\"DomMatrix\",\"DomMatrixReadOnly\"],\"WebSocket\":[\"EventTarget\"],\"WebSocketDict\":[],\"WebSocketElement\":[],\"WebTransport\":[],\"WebTransportBidirectionalStream\":[],\"WebTransportCloseInfo\":[],\"WebTransportCongestionControl\":[],\"WebTransportDatagramDuplexStream\":[],\"WebTransportDatagramStats\":[],\"WebTransportError\":[\"DomException\"],\"WebTransportErrorOptions\":[],\"WebTransportErrorSource\":[],\"WebTransportHash\":[],\"WebTransportOptions\":[],\"WebTransportReceiveStream\":[\"ReadableStream\"],\"WebTransportReceiveStreamStats\":[],\"WebTransportReliabilityMode\":[],\"WebTransportSendStream\":[\"WritableStream\"],\"WebTransportSendStreamOptions\":[],\"WebTransportSendStreamStats\":[],\"WebTransportStats\":[],\"WebglColorBufferFloat\":[],\"WebglCompressedTextureAstc\":[],\"WebglCompressedTextureAtc\":[],\"WebglCompressedTextureEtc\":[],\"WebglCompressedTextureEtc1\":[],\"WebglCompressedTexturePvrtc\":[],\"WebglCompressedTextureS3tc\":[],\"WebglCompressedTextureS3tcSrgb\":[],\"WebglDebugRendererInfo\":[],\"WebglDebugShaders\":[],\"WebglDepthTexture\":[],\"WebglDrawBuffers\":[],\"WebglLoseContext\":[],\"WebglMultiDraw\":[],\"WellKnownDirectory\":[],\"WgslLanguageFeatures\":[],\"WheelEvent\":[\"Event\",\"MouseEvent\",\"UiEvent\"],\"WheelEventInit\":[],\"WidevineCdmManifest\":[],\"Window\":[\"EventTarget\"],\"WindowClient\":[\"Client\"],\"Worker\":[\"EventTarget\"],\"WorkerDebuggerGlobalScope\":[\"EventTarget\"],\"WorkerGlobalScope\":[\"EventTarget\"],\"WorkerLocation\":[],\"WorkerNavigator\":[],\"WorkerOptions\":[],\"WorkerType\":[],\"Worklet\":[],\"WorkletGlobalScope\":[],\"WorkletOptions\":[],\"WritableStream\":[],\"WritableStreamDefaultController\":[],\"WritableStreamDefaultWriter\":[],\"WriteCommandType\":[],\"WriteParams\":[],\"XPathExpression\":[],\"XPathNsResolver\":[],\"XPathResult\":[],\"XmlDocument\":[\"Document\",\"EventTarget\",\"Node\"],\"XmlHttpRequest\":[\"EventTarget\",\"XmlHttpRequestEventTarget\"],\"XmlHttpRequestEventTarget\":[\"EventTarget\"],\"XmlHttpRequestResponseType\":[],\"XmlHttpRequestUpload\":[\"EventTarget\",\"XmlHttpRequestEventTarget\"],\"XmlSerializer\":[],\"XrBoundedReferenceSpace\":[\"EventTarget\",\"XrReferenceSpace\",\"XrSpace\"],\"XrEye\":[],\"XrFrame\":[],\"XrHand\":[],\"XrHandJoint\":[],\"XrHandedness\":[],\"XrInputSource\":[],\"XrInputSourceArray\":[],\"XrInputSourceEvent\":[\"Event\"],\"XrInputSourceEventInit\":[],\"XrInputSourcesChangeEvent\":[\"Event\"],\"XrInputSourcesChangeEventInit\":[],\"XrJointPose\":[\"XrPose\"],\"XrJointSpace\":[\"EventTarget\",\"XrSpace\"],\"XrLayer\":[\"EventTarget\"],\"XrPermissionDescriptor\":[],\"XrPermissionStatus\":[\"EventTarget\",\"PermissionStatus\"],\"XrPose\":[],\"XrReferenceSpace\":[\"EventTarget\",\"XrSpace\"],\"XrReferenceSpaceEvent\":[\"Event\"],\"XrReferenceSpaceEventInit\":[],\"XrReferenceSpaceType\":[],\"XrRenderState\":[],\"XrRenderStateInit\":[],\"XrRigidTransform\":[],\"XrSession\":[\"EventTarget\"],\"XrSessionEvent\":[\"Event\"],\"XrSessionEventInit\":[],\"XrSessionInit\":[],\"XrSessionMode\":[],\"XrSessionSupportedPermissionDescriptor\":[],\"XrSpace\":[\"EventTarget\"],\"XrSystem\":[\"EventTarget\"],\"XrTargetRayMode\":[],\"XrView\":[],\"XrViewerPose\":[\"XrPose\"],\"XrViewport\":[],\"XrVisibilityState\":[],\"XrWebGlLayer\":[\"EventTarget\",\"XrLayer\"],\"XrWebGlLayerInit\":[],\"XsltProcessor\":[],\"console\":[],\"css\":[],\"default\":[\"std\"],\"gpu_buffer_usage\":[],\"gpu_color_write\":[],\"gpu_map_mode\":[],\"gpu_shader_stage\":[],\"gpu_texture_usage\":[],\"std\":[\"wasm-bindgen/std\",\"js-sys/std\"]}}", + "web-sys_0.3.82": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"futures\",\"req\":\"^0.3\",\"target\":\"cfg(target_arch = \\\"wasm32\\\")\"},{\"default_features\":false,\"name\":\"js-sys\",\"req\":\"=0.3.82\"},{\"default_features\":false,\"name\":\"wasm-bindgen\",\"req\":\"=0.2.105\"}],\"features\":{\"AbortController\":[],\"AbortSignal\":[\"EventTarget\"],\"AddEventListenerOptions\":[],\"AesCbcParams\":[],\"AesCtrParams\":[],\"AesDerivedKeyParams\":[],\"AesGcmParams\":[],\"AesKeyAlgorithm\":[],\"AesKeyGenParams\":[],\"Algorithm\":[],\"AlignSetting\":[],\"AllowedBluetoothDevice\":[],\"AllowedUsbDevice\":[],\"AlphaOption\":[],\"AnalyserNode\":[\"AudioNode\",\"EventTarget\"],\"AnalyserOptions\":[],\"AngleInstancedArrays\":[],\"Animation\":[\"EventTarget\"],\"AnimationEffect\":[],\"AnimationEvent\":[\"Event\"],\"AnimationEventInit\":[],\"AnimationPlayState\":[],\"AnimationPlaybackEvent\":[\"Event\"],\"AnimationPlaybackEventInit\":[],\"AnimationPropertyDetails\":[],\"AnimationPropertyValueDetails\":[],\"AnimationTimeline\":[],\"AssignedNodesOptions\":[],\"AttestationConveyancePreference\":[],\"Attr\":[\"EventTarget\",\"Node\"],\"AttributeNameValue\":[],\"AudioBuffer\":[],\"AudioBufferOptions\":[],\"AudioBufferSourceNode\":[\"AudioNode\",\"AudioScheduledSourceNode\",\"EventTarget\"],\"AudioBufferSourceOptions\":[],\"AudioConfiguration\":[],\"AudioContext\":[\"BaseAudioContext\",\"EventTarget\"],\"AudioContextLatencyCategory\":[],\"AudioContextOptions\":[],\"AudioContextState\":[],\"AudioData\":[],\"AudioDataCopyToOptions\":[],\"AudioDataInit\":[],\"AudioDecoder\":[],\"AudioDecoderConfig\":[],\"AudioDecoderInit\":[],\"AudioDecoderSupport\":[],\"AudioDestinationNode\":[\"AudioNode\",\"EventTarget\"],\"AudioEncoder\":[],\"AudioEncoderConfig\":[],\"AudioEncoderInit\":[],\"AudioEncoderSupport\":[],\"AudioListener\":[],\"AudioNode\":[\"EventTarget\"],\"AudioNodeOptions\":[],\"AudioParam\":[],\"AudioParamMap\":[],\"AudioProcessingEvent\":[\"Event\"],\"AudioSampleFormat\":[],\"AudioScheduledSourceNode\":[\"AudioNode\",\"EventTarget\"],\"AudioSinkInfo\":[],\"AudioSinkOptions\":[],\"AudioSinkType\":[],\"AudioStreamTrack\":[\"EventTarget\",\"MediaStreamTrack\"],\"AudioTrack\":[],\"AudioTrackList\":[\"EventTarget\"],\"AudioWorklet\":[\"Worklet\"],\"AudioWorkletGlobalScope\":[\"WorkletGlobalScope\"],\"AudioWorkletNode\":[\"AudioNode\",\"EventTarget\"],\"AudioWorkletNodeOptions\":[],\"AudioWorkletProcessor\":[],\"AuthenticationExtensionsClientInputs\":[],\"AuthenticationExtensionsClientInputsJson\":[],\"AuthenticationExtensionsClientOutputs\":[],\"AuthenticationExtensionsClientOutputsJson\":[],\"AuthenticationExtensionsDevicePublicKeyInputs\":[],\"AuthenticationExtensionsDevicePublicKeyOutputs\":[],\"AuthenticationExtensionsLargeBlobInputs\":[],\"AuthenticationExtensionsLargeBlobOutputs\":[],\"AuthenticationExtensionsPrfInputs\":[],\"AuthenticationExtensionsPrfOutputs\":[],\"AuthenticationExtensionsPrfValues\":[],\"AuthenticationResponseJson\":[],\"AuthenticatorAssertionResponse\":[\"AuthenticatorResponse\"],\"AuthenticatorAssertionResponseJson\":[],\"AuthenticatorAttachment\":[],\"AuthenticatorAttestationResponse\":[\"AuthenticatorResponse\"],\"AuthenticatorAttestationResponseJson\":[],\"AuthenticatorResponse\":[],\"AuthenticatorSelectionCriteria\":[],\"AuthenticatorTransport\":[],\"AutoKeyword\":[],\"AutocompleteInfo\":[],\"BarProp\":[],\"BaseAudioContext\":[\"EventTarget\"],\"BaseComputedKeyframe\":[],\"BaseKeyframe\":[],\"BasePropertyIndexedKeyframe\":[],\"BasicCardRequest\":[],\"BasicCardResponse\":[],\"BasicCardType\":[],\"BatteryManager\":[\"EventTarget\"],\"BeforeUnloadEvent\":[\"Event\"],\"BinaryType\":[],\"BiquadFilterNode\":[\"AudioNode\",\"EventTarget\"],\"BiquadFilterOptions\":[],\"BiquadFilterType\":[],\"Blob\":[],\"BlobEvent\":[\"Event\"],\"BlobEventInit\":[],\"BlobPropertyBag\":[],\"BlockParsingOptions\":[],\"Bluetooth\":[\"EventTarget\"],\"BluetoothAdvertisingEvent\":[\"Event\"],\"BluetoothAdvertisingEventInit\":[],\"BluetoothCharacteristicProperties\":[],\"BluetoothDataFilterInit\":[],\"BluetoothDevice\":[\"EventTarget\"],\"BluetoothLeScanFilterInit\":[],\"BluetoothManufacturerDataMap\":[],\"BluetoothPermissionDescriptor\":[],\"BluetoothPermissionResult\":[\"EventTarget\",\"PermissionStatus\"],\"BluetoothPermissionStorage\":[],\"BluetoothRemoteGattCharacteristic\":[\"EventTarget\"],\"BluetoothRemoteGattDescriptor\":[],\"BluetoothRemoteGattServer\":[],\"BluetoothRemoteGattService\":[\"EventTarget\"],\"BluetoothServiceDataMap\":[],\"BluetoothUuid\":[],\"BoxQuadOptions\":[],\"BroadcastChannel\":[\"EventTarget\"],\"BrowserElementDownloadOptions\":[],\"BrowserElementExecuteScriptOptions\":[],\"BrowserFeedWriter\":[],\"BrowserFindCaseSensitivity\":[],\"BrowserFindDirection\":[],\"ByteLengthQueuingStrategy\":[],\"Cache\":[],\"CacheBatchOperation\":[],\"CacheQueryOptions\":[],\"CacheStorage\":[],\"CacheStorageNamespace\":[],\"CanvasCaptureMediaStream\":[\"EventTarget\",\"MediaStream\"],\"CanvasCaptureMediaStreamTrack\":[\"EventTarget\",\"MediaStreamTrack\"],\"CanvasGradient\":[],\"CanvasPattern\":[],\"CanvasRenderingContext2d\":[],\"CanvasWindingRule\":[],\"CaretChangedReason\":[],\"CaretPosition\":[],\"CaretStateChangedEventInit\":[],\"CdataSection\":[\"CharacterData\",\"EventTarget\",\"Node\",\"Text\"],\"ChannelCountMode\":[],\"ChannelInterpretation\":[],\"ChannelMergerNode\":[\"AudioNode\",\"EventTarget\"],\"ChannelMergerOptions\":[],\"ChannelSplitterNode\":[\"AudioNode\",\"EventTarget\"],\"ChannelSplitterOptions\":[],\"CharacterData\":[\"EventTarget\",\"Node\"],\"CheckerboardReason\":[],\"CheckerboardReport\":[],\"CheckerboardReportService\":[],\"ChromeFilePropertyBag\":[],\"ChromeWorker\":[\"EventTarget\",\"Worker\"],\"Client\":[],\"ClientQueryOptions\":[],\"ClientRectsAndTexts\":[],\"ClientType\":[],\"Clients\":[],\"Clipboard\":[\"EventTarget\"],\"ClipboardEvent\":[\"Event\"],\"ClipboardEventInit\":[],\"ClipboardItem\":[],\"ClipboardItemOptions\":[],\"ClipboardPermissionDescriptor\":[],\"ClipboardUnsanitizedFormats\":[],\"CloseEvent\":[\"Event\"],\"CloseEventInit\":[],\"CodecState\":[],\"CollectedClientData\":[],\"ColorSpaceConversion\":[],\"Comment\":[\"CharacterData\",\"EventTarget\",\"Node\"],\"CompositeOperation\":[],\"CompositionEvent\":[\"Event\",\"UiEvent\"],\"CompositionEventInit\":[],\"CompressionFormat\":[],\"CompressionStream\":[],\"ComputedEffectTiming\":[],\"ConnStatusDict\":[],\"ConnectionType\":[],\"ConsoleCounter\":[],\"ConsoleCounterError\":[],\"ConsoleEvent\":[],\"ConsoleInstance\":[],\"ConsoleInstanceOptions\":[],\"ConsoleLevel\":[],\"ConsoleLogLevel\":[],\"ConsoleProfileEvent\":[],\"ConsoleStackEntry\":[],\"ConsoleTimerError\":[],\"ConsoleTimerLogOrEnd\":[],\"ConsoleTimerStart\":[],\"ConstantSourceNode\":[\"AudioNode\",\"AudioScheduledSourceNode\",\"EventTarget\"],\"ConstantSourceOptions\":[],\"ConstrainBooleanParameters\":[],\"ConstrainDomStringParameters\":[],\"ConstrainDoubleRange\":[],\"ConstrainLongRange\":[],\"ContextAttributes2d\":[],\"ConvertCoordinateOptions\":[],\"ConvolverNode\":[\"AudioNode\",\"EventTarget\"],\"ConvolverOptions\":[],\"CookieChangeEvent\":[\"Event\"],\"CookieChangeEventInit\":[],\"CookieInit\":[],\"CookieListItem\":[],\"CookieSameSite\":[],\"CookieStore\":[\"EventTarget\"],\"CookieStoreDeleteOptions\":[],\"CookieStoreGetOptions\":[],\"CookieStoreManager\":[],\"Coordinates\":[],\"CountQueuingStrategy\":[],\"Credential\":[],\"CredentialCreationOptions\":[],\"CredentialPropertiesOutput\":[],\"CredentialRequestOptions\":[],\"CredentialsContainer\":[],\"Crypto\":[],\"CryptoKey\":[],\"CryptoKeyPair\":[],\"CssAnimation\":[\"Animation\",\"EventTarget\"],\"CssBoxType\":[],\"CssConditionRule\":[\"CssGroupingRule\",\"CssRule\"],\"CssCounterStyleRule\":[\"CssRule\"],\"CssFontFaceRule\":[\"CssRule\"],\"CssFontFeatureValuesRule\":[\"CssRule\"],\"CssGroupingRule\":[\"CssRule\"],\"CssImportRule\":[\"CssRule\"],\"CssKeyframeRule\":[\"CssRule\"],\"CssKeyframesRule\":[\"CssRule\"],\"CssMediaRule\":[\"CssConditionRule\",\"CssGroupingRule\",\"CssRule\"],\"CssNamespaceRule\":[\"CssRule\"],\"CssPageRule\":[\"CssRule\"],\"CssPseudoElement\":[],\"CssRule\":[],\"CssRuleList\":[],\"CssStyleDeclaration\":[],\"CssStyleRule\":[\"CssRule\"],\"CssStyleSheet\":[\"StyleSheet\"],\"CssStyleSheetParsingMode\":[],\"CssSupportsRule\":[\"CssConditionRule\",\"CssGroupingRule\",\"CssRule\"],\"CssTransition\":[\"Animation\",\"EventTarget\"],\"CustomElementRegistry\":[],\"CustomEvent\":[\"Event\"],\"CustomEventInit\":[],\"DataTransfer\":[],\"DataTransferItem\":[],\"DataTransferItemList\":[],\"DateTimeValue\":[],\"DecoderDoctorNotification\":[],\"DecoderDoctorNotificationType\":[],\"DecompressionStream\":[],\"DedicatedWorkerGlobalScope\":[\"EventTarget\",\"WorkerGlobalScope\"],\"DelayNode\":[\"AudioNode\",\"EventTarget\"],\"DelayOptions\":[],\"DeviceAcceleration\":[],\"DeviceAccelerationInit\":[],\"DeviceLightEvent\":[\"Event\"],\"DeviceLightEventInit\":[],\"DeviceMotionEvent\":[\"Event\"],\"DeviceMotionEventInit\":[],\"DeviceOrientationEvent\":[\"Event\"],\"DeviceOrientationEventInit\":[],\"DeviceProximityEvent\":[\"Event\"],\"DeviceProximityEventInit\":[],\"DeviceRotationRate\":[],\"DeviceRotationRateInit\":[],\"DhKeyDeriveParams\":[],\"DirectionSetting\":[],\"Directory\":[],\"DirectoryPickerOptions\":[],\"DisplayMediaStreamConstraints\":[],\"DisplayNameOptions\":[],\"DisplayNameResult\":[],\"DistanceModelType\":[],\"DnsCacheDict\":[],\"DnsCacheEntry\":[],\"DnsLookupDict\":[],\"Document\":[\"EventTarget\",\"Node\"],\"DocumentFragment\":[\"EventTarget\",\"Node\"],\"DocumentTimeline\":[\"AnimationTimeline\"],\"DocumentTimelineOptions\":[],\"DocumentType\":[\"EventTarget\",\"Node\"],\"DomError\":[],\"DomException\":[],\"DomImplementation\":[],\"DomMatrix\":[\"DomMatrixReadOnly\"],\"DomMatrix2dInit\":[],\"DomMatrixInit\":[],\"DomMatrixReadOnly\":[],\"DomParser\":[],\"DomPoint\":[\"DomPointReadOnly\"],\"DomPointInit\":[],\"DomPointReadOnly\":[],\"DomQuad\":[],\"DomQuadInit\":[],\"DomQuadJson\":[],\"DomRect\":[\"DomRectReadOnly\"],\"DomRectInit\":[],\"DomRectList\":[],\"DomRectReadOnly\":[],\"DomRequest\":[\"EventTarget\"],\"DomRequestReadyState\":[],\"DomStringList\":[],\"DomStringMap\":[],\"DomTokenList\":[],\"DomWindowResizeEventDetail\":[],\"DoubleRange\":[],\"DragEvent\":[\"Event\",\"MouseEvent\",\"UiEvent\"],\"DragEventInit\":[],\"DynamicsCompressorNode\":[\"AudioNode\",\"EventTarget\"],\"DynamicsCompressorOptions\":[],\"EcKeyAlgorithm\":[],\"EcKeyGenParams\":[],\"EcKeyImportParams\":[],\"EcdhKeyDeriveParams\":[],\"EcdsaParams\":[],\"EffectTiming\":[],\"Element\":[\"EventTarget\",\"Node\"],\"ElementCreationOptions\":[],\"ElementDefinitionOptions\":[],\"EncodedAudioChunk\":[],\"EncodedAudioChunkInit\":[],\"EncodedAudioChunkMetadata\":[],\"EncodedAudioChunkType\":[],\"EncodedVideoChunk\":[],\"EncodedVideoChunkInit\":[],\"EncodedVideoChunkMetadata\":[],\"EncodedVideoChunkType\":[],\"EndingTypes\":[],\"ErrorCallback\":[],\"ErrorEvent\":[\"Event\"],\"ErrorEventInit\":[],\"Event\":[],\"EventInit\":[],\"EventListener\":[],\"EventListenerOptions\":[],\"EventModifierInit\":[],\"EventSource\":[\"EventTarget\"],\"EventSourceInit\":[],\"EventTarget\":[],\"Exception\":[],\"ExtBlendMinmax\":[],\"ExtColorBufferFloat\":[],\"ExtColorBufferHalfFloat\":[],\"ExtDisjointTimerQuery\":[],\"ExtFragDepth\":[],\"ExtSRgb\":[],\"ExtShaderTextureLod\":[],\"ExtTextureFilterAnisotropic\":[],\"ExtTextureNorm16\":[],\"ExtendableCookieChangeEvent\":[\"Event\",\"ExtendableEvent\"],\"ExtendableCookieChangeEventInit\":[],\"ExtendableEvent\":[\"Event\"],\"ExtendableEventInit\":[],\"ExtendableMessageEvent\":[\"Event\",\"ExtendableEvent\"],\"ExtendableMessageEventInit\":[],\"External\":[],\"FakePluginMimeEntry\":[],\"FakePluginTagInit\":[],\"FetchEvent\":[\"Event\",\"ExtendableEvent\"],\"FetchEventInit\":[],\"FetchObserver\":[\"EventTarget\"],\"FetchReadableStreamReadDataArray\":[],\"FetchReadableStreamReadDataDone\":[],\"FetchState\":[],\"File\":[\"Blob\"],\"FileCallback\":[],\"FileList\":[],\"FilePickerAcceptType\":[],\"FilePickerOptions\":[],\"FilePropertyBag\":[],\"FileReader\":[\"EventTarget\"],\"FileReaderSync\":[],\"FileSystem\":[],\"FileSystemCreateWritableOptions\":[],\"FileSystemDirectoryEntry\":[\"FileSystemEntry\"],\"FileSystemDirectoryHandle\":[\"FileSystemHandle\"],\"FileSystemDirectoryReader\":[],\"FileSystemEntriesCallback\":[],\"FileSystemEntry\":[],\"FileSystemEntryCallback\":[],\"FileSystemFileEntry\":[\"FileSystemEntry\"],\"FileSystemFileHandle\":[\"FileSystemHandle\"],\"FileSystemFlags\":[],\"FileSystemGetDirectoryOptions\":[],\"FileSystemGetFileOptions\":[],\"FileSystemHandle\":[],\"FileSystemHandleKind\":[],\"FileSystemHandlePermissionDescriptor\":[],\"FileSystemPermissionDescriptor\":[],\"FileSystemPermissionMode\":[],\"FileSystemReadWriteOptions\":[],\"FileSystemRemoveOptions\":[],\"FileSystemSyncAccessHandle\":[],\"FileSystemWritableFileStream\":[\"WritableStream\"],\"FillMode\":[],\"FlashClassification\":[],\"FlowControlType\":[],\"FocusEvent\":[\"Event\",\"UiEvent\"],\"FocusEventInit\":[],\"FocusOptions\":[],\"FontData\":[],\"FontFace\":[],\"FontFaceDescriptors\":[],\"FontFaceLoadStatus\":[],\"FontFaceSet\":[\"EventTarget\"],\"FontFaceSetIterator\":[],\"FontFaceSetIteratorResult\":[],\"FontFaceSetLoadEvent\":[\"Event\"],\"FontFaceSetLoadEventInit\":[],\"FontFaceSetLoadStatus\":[],\"FormData\":[],\"FrameType\":[],\"FuzzingFunctions\":[],\"GainNode\":[\"AudioNode\",\"EventTarget\"],\"GainOptions\":[],\"Gamepad\":[],\"GamepadButton\":[],\"GamepadEffectParameters\":[],\"GamepadEvent\":[\"Event\"],\"GamepadEventInit\":[],\"GamepadHand\":[],\"GamepadHapticActuator\":[],\"GamepadHapticActuatorType\":[],\"GamepadHapticEffectType\":[],\"GamepadHapticsResult\":[],\"GamepadMappingType\":[],\"GamepadPose\":[],\"GamepadTouch\":[],\"Geolocation\":[],\"GestureEvent\":[\"Event\",\"UiEvent\"],\"GetAnimationsOptions\":[],\"GetRootNodeOptions\":[],\"GetUserMediaRequest\":[],\"Gpu\":[],\"GpuAdapter\":[],\"GpuAdapterInfo\":[],\"GpuAddressMode\":[],\"GpuAutoLayoutMode\":[],\"GpuBindGroup\":[],\"GpuBindGroupDescriptor\":[],\"GpuBindGroupEntry\":[],\"GpuBindGroupLayout\":[],\"GpuBindGroupLayoutDescriptor\":[],\"GpuBindGroupLayoutEntry\":[],\"GpuBlendComponent\":[],\"GpuBlendFactor\":[],\"GpuBlendOperation\":[],\"GpuBlendState\":[],\"GpuBuffer\":[],\"GpuBufferBinding\":[],\"GpuBufferBindingLayout\":[],\"GpuBufferBindingType\":[],\"GpuBufferDescriptor\":[],\"GpuBufferMapState\":[],\"GpuCanvasAlphaMode\":[],\"GpuCanvasConfiguration\":[],\"GpuCanvasContext\":[],\"GpuCanvasToneMapping\":[],\"GpuCanvasToneMappingMode\":[],\"GpuColorDict\":[],\"GpuColorTargetState\":[],\"GpuCommandBuffer\":[],\"GpuCommandBufferDescriptor\":[],\"GpuCommandEncoder\":[],\"GpuCommandEncoderDescriptor\":[],\"GpuCompareFunction\":[],\"GpuCompilationInfo\":[],\"GpuCompilationMessage\":[],\"GpuCompilationMessageType\":[],\"GpuComputePassDescriptor\":[],\"GpuComputePassEncoder\":[],\"GpuComputePassTimestampWrites\":[],\"GpuComputePipeline\":[],\"GpuComputePipelineDescriptor\":[],\"GpuCopyExternalImageDestInfo\":[],\"GpuCopyExternalImageSourceInfo\":[],\"GpuCullMode\":[],\"GpuDepthStencilState\":[],\"GpuDevice\":[\"EventTarget\"],\"GpuDeviceDescriptor\":[],\"GpuDeviceLostInfo\":[],\"GpuDeviceLostReason\":[],\"GpuError\":[],\"GpuErrorFilter\":[],\"GpuExtent3dDict\":[],\"GpuExternalTexture\":[],\"GpuExternalTextureBindingLayout\":[],\"GpuExternalTextureDescriptor\":[],\"GpuFeatureName\":[],\"GpuFilterMode\":[],\"GpuFragmentState\":[],\"GpuFrontFace\":[],\"GpuIndexFormat\":[],\"GpuInternalError\":[\"GpuError\"],\"GpuLoadOp\":[],\"GpuMipmapFilterMode\":[],\"GpuMultisampleState\":[],\"GpuObjectDescriptorBase\":[],\"GpuOrigin2dDict\":[],\"GpuOrigin3dDict\":[],\"GpuOutOfMemoryError\":[\"GpuError\"],\"GpuPipelineDescriptorBase\":[],\"GpuPipelineError\":[\"DomException\"],\"GpuPipelineErrorInit\":[],\"GpuPipelineErrorReason\":[],\"GpuPipelineLayout\":[],\"GpuPipelineLayoutDescriptor\":[],\"GpuPowerPreference\":[],\"GpuPrimitiveState\":[],\"GpuPrimitiveTopology\":[],\"GpuProgrammableStage\":[],\"GpuQuerySet\":[],\"GpuQuerySetDescriptor\":[],\"GpuQueryType\":[],\"GpuQueue\":[],\"GpuQueueDescriptor\":[],\"GpuRenderBundle\":[],\"GpuRenderBundleDescriptor\":[],\"GpuRenderBundleEncoder\":[],\"GpuRenderBundleEncoderDescriptor\":[],\"GpuRenderPassColorAttachment\":[],\"GpuRenderPassDepthStencilAttachment\":[],\"GpuRenderPassDescriptor\":[],\"GpuRenderPassEncoder\":[],\"GpuRenderPassLayout\":[],\"GpuRenderPassTimestampWrites\":[],\"GpuRenderPipeline\":[],\"GpuRenderPipelineDescriptor\":[],\"GpuRequestAdapterOptions\":[],\"GpuSampler\":[],\"GpuSamplerBindingLayout\":[],\"GpuSamplerBindingType\":[],\"GpuSamplerDescriptor\":[],\"GpuShaderModule\":[],\"GpuShaderModuleCompilationHint\":[],\"GpuShaderModuleDescriptor\":[],\"GpuStencilFaceState\":[],\"GpuStencilOperation\":[],\"GpuStorageTextureAccess\":[],\"GpuStorageTextureBindingLayout\":[],\"GpuStoreOp\":[],\"GpuSupportedFeatures\":[],\"GpuSupportedLimits\":[],\"GpuTexelCopyBufferInfo\":[],\"GpuTexelCopyBufferLayout\":[],\"GpuTexelCopyTextureInfo\":[],\"GpuTexture\":[],\"GpuTextureAspect\":[],\"GpuTextureBindingLayout\":[],\"GpuTextureDescriptor\":[],\"GpuTextureDimension\":[],\"GpuTextureFormat\":[],\"GpuTextureSampleType\":[],\"GpuTextureView\":[],\"GpuTextureViewDescriptor\":[],\"GpuTextureViewDimension\":[],\"GpuUncapturedErrorEvent\":[\"Event\"],\"GpuUncapturedErrorEventInit\":[],\"GpuValidationError\":[\"GpuError\"],\"GpuVertexAttribute\":[],\"GpuVertexBufferLayout\":[],\"GpuVertexFormat\":[],\"GpuVertexState\":[],\"GpuVertexStepMode\":[],\"GroupedHistoryEventInit\":[],\"HalfOpenInfoDict\":[],\"HardwareAcceleration\":[],\"HashChangeEvent\":[\"Event\"],\"HashChangeEventInit\":[],\"Headers\":[],\"HeadersGuardEnum\":[],\"Hid\":[\"EventTarget\"],\"HidCollectionInfo\":[],\"HidConnectionEvent\":[\"Event\"],\"HidConnectionEventInit\":[],\"HidDevice\":[\"EventTarget\"],\"HidDeviceFilter\":[],\"HidDeviceRequestOptions\":[],\"HidInputReportEvent\":[\"Event\"],\"HidInputReportEventInit\":[],\"HidReportInfo\":[],\"HidReportItem\":[],\"HidUnitSystem\":[],\"HiddenPluginEventInit\":[],\"History\":[],\"HitRegionOptions\":[],\"HkdfParams\":[],\"HmacDerivedKeyParams\":[],\"HmacImportParams\":[],\"HmacKeyAlgorithm\":[],\"HmacKeyGenParams\":[],\"HtmlAllCollection\":[],\"HtmlAnchorElement\":[\"Element\",\"EventTarget\",\"HtmlElement\",\"Node\"],\"HtmlAreaElement\":[\"Element\",\"EventTarget\",\"HtmlElement\",\"Node\"],\"HtmlAudioElement\":[\"Element\",\"EventTarget\",\"HtmlElement\",\"HtmlMediaElement\",\"Node\"],\"HtmlBaseElement\":[\"Element\",\"EventTarget\",\"HtmlElement\",\"Node\"],\"HtmlBodyElement\":[\"Element\",\"EventTarget\",\"HtmlElement\",\"Node\"],\"HtmlBrElement\":[\"Element\",\"EventTarget\",\"HtmlElement\",\"Node\"],\"HtmlButtonElement\":[\"Element\",\"EventTarget\",\"HtmlElement\",\"Node\"],\"HtmlCanvasElement\":[\"Element\",\"EventTarget\",\"HtmlElement\",\"Node\"],\"HtmlCollection\":[],\"HtmlDListElement\":[\"Element\",\"EventTarget\",\"HtmlElement\",\"Node\"],\"HtmlDataElement\":[\"Element\",\"EventTarget\",\"HtmlElement\",\"Node\"],\"HtmlDataListElement\":[\"Element\",\"EventTarget\",\"HtmlElement\",\"Node\"],\"HtmlDetailsElement\":[\"Element\",\"EventTarget\",\"HtmlElement\",\"Node\"],\"HtmlDialogElement\":[\"Element\",\"EventTarget\",\"HtmlElement\",\"Node\"],\"HtmlDirectoryElement\":[\"Element\",\"EventTarget\",\"HtmlElement\",\"Node\"],\"HtmlDivElement\":[\"Element\",\"EventTarget\",\"HtmlElement\",\"Node\"],\"HtmlDocument\":[\"Document\",\"EventTarget\",\"Node\"],\"HtmlElement\":[\"Element\",\"EventTarget\",\"Node\"],\"HtmlEmbedElement\":[\"Element\",\"EventTarget\",\"HtmlElement\",\"Node\"],\"HtmlFieldSetElement\":[\"Element\",\"EventTarget\",\"HtmlElement\",\"Node\"],\"HtmlFontElement\":[\"Element\",\"EventTarget\",\"HtmlElement\",\"Node\"],\"HtmlFormControlsCollection\":[\"HtmlCollection\"],\"HtmlFormElement\":[\"Element\",\"EventTarget\",\"HtmlElement\",\"Node\"],\"HtmlFrameElement\":[\"Element\",\"EventTarget\",\"HtmlElement\",\"Node\"],\"HtmlFrameSetElement\":[\"Element\",\"EventTarget\",\"HtmlElement\",\"Node\"],\"HtmlHeadElement\":[\"Element\",\"EventTarget\",\"HtmlElement\",\"Node\"],\"HtmlHeadingElement\":[\"Element\",\"EventTarget\",\"HtmlElement\",\"Node\"],\"HtmlHrElement\":[\"Element\",\"EventTarget\",\"HtmlElement\",\"Node\"],\"HtmlHtmlElement\":[\"Element\",\"EventTarget\",\"HtmlElement\",\"Node\"],\"HtmlIFrameElement\":[\"Element\",\"EventTarget\",\"HtmlElement\",\"Node\"],\"HtmlImageElement\":[\"Element\",\"EventTarget\",\"HtmlElement\",\"Node\"],\"HtmlInputElement\":[\"Element\",\"EventTarget\",\"HtmlElement\",\"Node\"],\"HtmlLabelElement\":[\"Element\",\"EventTarget\",\"HtmlElement\",\"Node\"],\"HtmlLegendElement\":[\"Element\",\"EventTarget\",\"HtmlElement\",\"Node\"],\"HtmlLiElement\":[\"Element\",\"EventTarget\",\"HtmlElement\",\"Node\"],\"HtmlLinkElement\":[\"Element\",\"EventTarget\",\"HtmlElement\",\"Node\"],\"HtmlMapElement\":[\"Element\",\"EventTarget\",\"HtmlElement\",\"Node\"],\"HtmlMediaElement\":[\"Element\",\"EventTarget\",\"HtmlElement\",\"Node\"],\"HtmlMenuElement\":[\"Element\",\"EventTarget\",\"HtmlElement\",\"Node\"],\"HtmlMenuItemElement\":[\"Element\",\"EventTarget\",\"HtmlElement\",\"Node\"],\"HtmlMetaElement\":[\"Element\",\"EventTarget\",\"HtmlElement\",\"Node\"],\"HtmlMeterElement\":[\"Element\",\"EventTarget\",\"HtmlElement\",\"Node\"],\"HtmlModElement\":[\"Element\",\"EventTarget\",\"HtmlElement\",\"Node\"],\"HtmlOListElement\":[\"Element\",\"EventTarget\",\"HtmlElement\",\"Node\"],\"HtmlObjectElement\":[\"Element\",\"EventTarget\",\"HtmlElement\",\"Node\"],\"HtmlOptGroupElement\":[\"Element\",\"EventTarget\",\"HtmlElement\",\"Node\"],\"HtmlOptionElement\":[\"Element\",\"EventTarget\",\"HtmlElement\",\"Node\"],\"HtmlOptionsCollection\":[\"HtmlCollection\"],\"HtmlOutputElement\":[\"Element\",\"EventTarget\",\"HtmlElement\",\"Node\"],\"HtmlParagraphElement\":[\"Element\",\"EventTarget\",\"HtmlElement\",\"Node\"],\"HtmlParamElement\":[\"Element\",\"EventTarget\",\"HtmlElement\",\"Node\"],\"HtmlPictureElement\":[\"Element\",\"EventTarget\",\"HtmlElement\",\"Node\"],\"HtmlPreElement\":[\"Element\",\"EventTarget\",\"HtmlElement\",\"Node\"],\"HtmlProgressElement\":[\"Element\",\"EventTarget\",\"HtmlElement\",\"Node\"],\"HtmlQuoteElement\":[\"Element\",\"EventTarget\",\"HtmlElement\",\"Node\"],\"HtmlScriptElement\":[\"Element\",\"EventTarget\",\"HtmlElement\",\"Node\"],\"HtmlSelectElement\":[\"Element\",\"EventTarget\",\"HtmlElement\",\"Node\"],\"HtmlSlotElement\":[\"Element\",\"EventTarget\",\"HtmlElement\",\"Node\"],\"HtmlSourceElement\":[\"Element\",\"EventTarget\",\"HtmlElement\",\"Node\"],\"HtmlSpanElement\":[\"Element\",\"EventTarget\",\"HtmlElement\",\"Node\"],\"HtmlStyleElement\":[\"Element\",\"EventTarget\",\"HtmlElement\",\"Node\"],\"HtmlTableCaptionElement\":[\"Element\",\"EventTarget\",\"HtmlElement\",\"Node\"],\"HtmlTableCellElement\":[\"Element\",\"EventTarget\",\"HtmlElement\",\"Node\"],\"HtmlTableColElement\":[\"Element\",\"EventTarget\",\"HtmlElement\",\"Node\"],\"HtmlTableElement\":[\"Element\",\"EventTarget\",\"HtmlElement\",\"Node\"],\"HtmlTableRowElement\":[\"Element\",\"EventTarget\",\"HtmlElement\",\"Node\"],\"HtmlTableSectionElement\":[\"Element\",\"EventTarget\",\"HtmlElement\",\"Node\"],\"HtmlTemplateElement\":[\"Element\",\"EventTarget\",\"HtmlElement\",\"Node\"],\"HtmlTextAreaElement\":[\"Element\",\"EventTarget\",\"HtmlElement\",\"Node\"],\"HtmlTimeElement\":[\"Element\",\"EventTarget\",\"HtmlElement\",\"Node\"],\"HtmlTitleElement\":[\"Element\",\"EventTarget\",\"HtmlElement\",\"Node\"],\"HtmlTrackElement\":[\"Element\",\"EventTarget\",\"HtmlElement\",\"Node\"],\"HtmlUListElement\":[\"Element\",\"EventTarget\",\"HtmlElement\",\"Node\"],\"HtmlUnknownElement\":[\"Element\",\"EventTarget\",\"HtmlElement\",\"Node\"],\"HtmlVideoElement\":[\"Element\",\"EventTarget\",\"HtmlElement\",\"HtmlMediaElement\",\"Node\"],\"HttpConnDict\":[],\"HttpConnInfo\":[],\"HttpConnectionElement\":[],\"IdbCursor\":[],\"IdbCursorDirection\":[],\"IdbCursorWithValue\":[\"IdbCursor\"],\"IdbDatabase\":[\"EventTarget\"],\"IdbFactory\":[],\"IdbFileHandle\":[\"EventTarget\"],\"IdbFileMetadataParameters\":[],\"IdbFileRequest\":[\"DomRequest\",\"EventTarget\"],\"IdbIndex\":[],\"IdbIndexParameters\":[],\"IdbKeyRange\":[],\"IdbLocaleAwareKeyRange\":[\"IdbKeyRange\"],\"IdbMutableFile\":[\"EventTarget\"],\"IdbObjectStore\":[],\"IdbObjectStoreParameters\":[],\"IdbOpenDbOptions\":[],\"IdbOpenDbRequest\":[\"EventTarget\",\"IdbRequest\"],\"IdbRequest\":[\"EventTarget\"],\"IdbRequestReadyState\":[],\"IdbTransaction\":[\"EventTarget\"],\"IdbTransactionDurability\":[],\"IdbTransactionMode\":[],\"IdbTransactionOptions\":[],\"IdbVersionChangeEvent\":[\"Event\"],\"IdbVersionChangeEventInit\":[],\"IdleDeadline\":[],\"IdleRequestOptions\":[],\"IirFilterNode\":[\"AudioNode\",\"EventTarget\"],\"IirFilterOptions\":[],\"ImageBitmap\":[],\"ImageBitmapOptions\":[],\"ImageBitmapRenderingContext\":[],\"ImageCapture\":[],\"ImageCaptureError\":[],\"ImageCaptureErrorEvent\":[\"Event\"],\"ImageCaptureErrorEventInit\":[],\"ImageData\":[],\"ImageDecodeOptions\":[],\"ImageDecodeResult\":[],\"ImageDecoder\":[],\"ImageDecoderInit\":[],\"ImageEncodeOptions\":[],\"ImageOrientation\":[],\"ImageTrack\":[\"EventTarget\"],\"ImageTrackList\":[],\"InputDeviceInfo\":[\"MediaDeviceInfo\"],\"InputEvent\":[\"Event\",\"UiEvent\"],\"InputEventInit\":[],\"IntersectionObserver\":[],\"IntersectionObserverEntry\":[],\"IntersectionObserverEntryInit\":[],\"IntersectionObserverInit\":[],\"IntlUtils\":[],\"IsInputPendingOptions\":[],\"IterableKeyAndValueResult\":[],\"IterableKeyOrValueResult\":[],\"IterationCompositeOperation\":[],\"JsonWebKey\":[],\"KeyAlgorithm\":[],\"KeyEvent\":[],\"KeyFrameRequestEvent\":[\"Event\"],\"KeyIdsInitData\":[],\"KeyboardEvent\":[\"Event\",\"UiEvent\"],\"KeyboardEventInit\":[],\"KeyframeAnimationOptions\":[],\"KeyframeEffect\":[\"AnimationEffect\"],\"KeyframeEffectOptions\":[],\"L10nElement\":[],\"L10nValue\":[],\"LargeBlobSupport\":[],\"LatencyMode\":[],\"LifecycleCallbacks\":[],\"LineAlignSetting\":[],\"ListBoxObject\":[],\"LocalMediaStream\":[\"EventTarget\",\"MediaStream\"],\"LocaleInfo\":[],\"Location\":[],\"Lock\":[],\"LockInfo\":[],\"LockManager\":[],\"LockManagerSnapshot\":[],\"LockMode\":[],\"LockOptions\":[],\"MathMlElement\":[\"Element\",\"EventTarget\",\"Node\"],\"MediaCapabilities\":[],\"MediaCapabilitiesInfo\":[],\"MediaConfiguration\":[],\"MediaDecodingConfiguration\":[],\"MediaDecodingType\":[],\"MediaDeviceInfo\":[],\"MediaDeviceKind\":[],\"MediaDevices\":[\"EventTarget\"],\"MediaElementAudioSourceNode\":[\"AudioNode\",\"EventTarget\"],\"MediaElementAudioSourceOptions\":[],\"MediaEncodingConfiguration\":[],\"MediaEncodingType\":[],\"MediaEncryptedEvent\":[\"Event\"],\"MediaError\":[],\"MediaImage\":[],\"MediaKeyError\":[\"Event\"],\"MediaKeyMessageEvent\":[\"Event\"],\"MediaKeyMessageEventInit\":[],\"MediaKeyMessageType\":[],\"MediaKeyNeededEventInit\":[],\"MediaKeySession\":[\"EventTarget\"],\"MediaKeySessionType\":[],\"MediaKeyStatus\":[],\"MediaKeyStatusMap\":[],\"MediaKeySystemAccess\":[],\"MediaKeySystemConfiguration\":[],\"MediaKeySystemMediaCapability\":[],\"MediaKeySystemStatus\":[],\"MediaKeys\":[],\"MediaKeysPolicy\":[],\"MediaKeysRequirement\":[],\"MediaList\":[],\"MediaMetadata\":[],\"MediaMetadataInit\":[],\"MediaPositionState\":[],\"MediaQueryList\":[\"EventTarget\"],\"MediaQueryListEvent\":[\"Event\"],\"MediaQueryListEventInit\":[],\"MediaRecorder\":[\"EventTarget\"],\"MediaRecorderErrorEvent\":[\"Event\"],\"MediaRecorderErrorEventInit\":[],\"MediaRecorderOptions\":[],\"MediaSession\":[],\"MediaSessionAction\":[],\"MediaSessionActionDetails\":[],\"MediaSessionPlaybackState\":[],\"MediaSource\":[\"EventTarget\"],\"MediaSourceEndOfStreamError\":[],\"MediaSourceEnum\":[],\"MediaSourceReadyState\":[],\"MediaStream\":[\"EventTarget\"],\"MediaStreamAudioDestinationNode\":[\"AudioNode\",\"EventTarget\"],\"MediaStreamAudioSourceNode\":[\"AudioNode\",\"EventTarget\"],\"MediaStreamAudioSourceOptions\":[],\"MediaStreamConstraints\":[],\"MediaStreamError\":[],\"MediaStreamEvent\":[\"Event\"],\"MediaStreamEventInit\":[],\"MediaStreamTrack\":[\"EventTarget\"],\"MediaStreamTrackEvent\":[\"Event\"],\"MediaStreamTrackEventInit\":[],\"MediaStreamTrackGenerator\":[\"EventTarget\",\"MediaStreamTrack\"],\"MediaStreamTrackGeneratorInit\":[],\"MediaStreamTrackProcessor\":[],\"MediaStreamTrackProcessorInit\":[],\"MediaStreamTrackState\":[],\"MediaTrackCapabilities\":[],\"MediaTrackConstraintSet\":[],\"MediaTrackConstraints\":[],\"MediaTrackSettings\":[],\"MediaTrackSupportedConstraints\":[],\"MemoryAttribution\":[],\"MemoryAttributionContainer\":[],\"MemoryBreakdownEntry\":[],\"MemoryMeasurement\":[],\"MessageChannel\":[],\"MessageEvent\":[\"Event\"],\"MessageEventInit\":[],\"MessagePort\":[\"EventTarget\"],\"MidiAccess\":[\"EventTarget\"],\"MidiConnectionEvent\":[\"Event\"],\"MidiConnectionEventInit\":[],\"MidiInput\":[\"EventTarget\",\"MidiPort\"],\"MidiInputMap\":[],\"MidiMessageEvent\":[\"Event\"],\"MidiMessageEventInit\":[],\"MidiOptions\":[],\"MidiOutput\":[\"EventTarget\",\"MidiPort\"],\"MidiOutputMap\":[],\"MidiPort\":[\"EventTarget\"],\"MidiPortConnectionState\":[],\"MidiPortDeviceState\":[],\"MidiPortType\":[],\"MimeType\":[],\"MimeTypeArray\":[],\"MouseEvent\":[\"Event\",\"UiEvent\"],\"MouseEventInit\":[],\"MouseScrollEvent\":[\"Event\",\"MouseEvent\",\"UiEvent\"],\"MozDebug\":[],\"MutationEvent\":[\"Event\"],\"MutationObserver\":[],\"MutationObserverInit\":[],\"MutationObservingInfo\":[],\"MutationRecord\":[],\"NamedNodeMap\":[],\"NativeOsFileReadOptions\":[],\"NativeOsFileWriteAtomicOptions\":[],\"NavigationType\":[],\"Navigator\":[],\"NavigatorAutomationInformation\":[],\"NavigatorUaBrandVersion\":[],\"NavigatorUaData\":[],\"NetworkCommandOptions\":[],\"NetworkInformation\":[\"EventTarget\"],\"NetworkResultOptions\":[],\"Node\":[\"EventTarget\"],\"NodeFilter\":[],\"NodeIterator\":[],\"NodeList\":[],\"Notification\":[\"EventTarget\"],\"NotificationAction\":[],\"NotificationDirection\":[],\"NotificationEvent\":[\"Event\",\"ExtendableEvent\"],\"NotificationEventInit\":[],\"NotificationOptions\":[],\"NotificationPermission\":[],\"ObserverCallback\":[],\"OesElementIndexUint\":[],\"OesStandardDerivatives\":[],\"OesTextureFloat\":[],\"OesTextureFloatLinear\":[],\"OesTextureHalfFloat\":[],\"OesTextureHalfFloatLinear\":[],\"OesVertexArrayObject\":[],\"OfflineAudioCompletionEvent\":[\"Event\"],\"OfflineAudioCompletionEventInit\":[],\"OfflineAudioContext\":[\"BaseAudioContext\",\"EventTarget\"],\"OfflineAudioContextOptions\":[],\"OfflineResourceList\":[\"EventTarget\"],\"OffscreenCanvas\":[\"EventTarget\"],\"OffscreenCanvasRenderingContext2d\":[],\"OpenFilePickerOptions\":[],\"OpenWindowEventDetail\":[],\"OptionalEffectTiming\":[],\"OrientationLockType\":[],\"OrientationType\":[],\"OscillatorNode\":[\"AudioNode\",\"AudioScheduledSourceNode\",\"EventTarget\"],\"OscillatorOptions\":[],\"OscillatorType\":[],\"OverSampleType\":[],\"OvrMultiview2\":[],\"PageTransitionEvent\":[\"Event\"],\"PageTransitionEventInit\":[],\"PaintRequest\":[],\"PaintRequestList\":[],\"PaintWorkletGlobalScope\":[\"WorkletGlobalScope\"],\"PannerNode\":[\"AudioNode\",\"EventTarget\"],\"PannerOptions\":[],\"PanningModelType\":[],\"ParityType\":[],\"Path2d\":[],\"PaymentAddress\":[],\"PaymentComplete\":[],\"PaymentMethodChangeEvent\":[\"Event\",\"PaymentRequestUpdateEvent\"],\"PaymentMethodChangeEventInit\":[],\"PaymentRequestUpdateEvent\":[\"Event\"],\"PaymentRequestUpdateEventInit\":[],\"PaymentResponse\":[],\"Pbkdf2Params\":[],\"PcImplIceConnectionState\":[],\"PcImplIceGatheringState\":[],\"PcImplSignalingState\":[],\"PcObserverStateType\":[],\"Performance\":[\"EventTarget\"],\"PerformanceEntry\":[],\"PerformanceEntryEventInit\":[],\"PerformanceEntryFilterOptions\":[],\"PerformanceMark\":[\"PerformanceEntry\"],\"PerformanceMeasure\":[\"PerformanceEntry\"],\"PerformanceNavigation\":[],\"PerformanceNavigationTiming\":[\"PerformanceEntry\",\"PerformanceResourceTiming\"],\"PerformanceObserver\":[],\"PerformanceObserverEntryList\":[],\"PerformanceObserverInit\":[],\"PerformanceResourceTiming\":[\"PerformanceEntry\"],\"PerformanceServerTiming\":[],\"PerformanceTiming\":[],\"PeriodicWave\":[],\"PeriodicWaveConstraints\":[],\"PeriodicWaveOptions\":[],\"PermissionDescriptor\":[],\"PermissionName\":[],\"PermissionState\":[],\"PermissionStatus\":[\"EventTarget\"],\"Permissions\":[],\"PictureInPictureEvent\":[\"Event\"],\"PictureInPictureEventInit\":[],\"PictureInPictureWindow\":[\"EventTarget\"],\"PlaneLayout\":[],\"PlaybackDirection\":[],\"Plugin\":[],\"PluginArray\":[],\"PluginCrashedEventInit\":[],\"PointerEvent\":[\"Event\",\"MouseEvent\",\"UiEvent\"],\"PointerEventInit\":[],\"PopStateEvent\":[\"Event\"],\"PopStateEventInit\":[],\"PopupBlockedEvent\":[\"Event\"],\"PopupBlockedEventInit\":[],\"Position\":[],\"PositionAlignSetting\":[],\"PositionError\":[],\"PositionOptions\":[],\"PremultiplyAlpha\":[],\"Presentation\":[],\"PresentationAvailability\":[\"EventTarget\"],\"PresentationConnection\":[\"EventTarget\"],\"PresentationConnectionAvailableEvent\":[\"Event\"],\"PresentationConnectionAvailableEventInit\":[],\"PresentationConnectionBinaryType\":[],\"PresentationConnectionCloseEvent\":[\"Event\"],\"PresentationConnectionCloseEventInit\":[],\"PresentationConnectionClosedReason\":[],\"PresentationConnectionList\":[\"EventTarget\"],\"PresentationConnectionState\":[],\"PresentationReceiver\":[],\"PresentationRequest\":[\"EventTarget\"],\"PresentationStyle\":[],\"ProcessingInstruction\":[\"CharacterData\",\"EventTarget\",\"Node\"],\"ProfileTimelineLayerRect\":[],\"ProfileTimelineMarker\":[],\"ProfileTimelineMessagePortOperationType\":[],\"ProfileTimelineStackFrame\":[],\"ProfileTimelineWorkerOperationType\":[],\"ProgressEvent\":[\"Event\"],\"ProgressEventInit\":[],\"PromiseNativeHandler\":[],\"PromiseRejectionEvent\":[\"Event\"],\"PromiseRejectionEventInit\":[],\"PublicKeyCredential\":[\"Credential\"],\"PublicKeyCredentialCreationOptions\":[],\"PublicKeyCredentialCreationOptionsJson\":[],\"PublicKeyCredentialDescriptor\":[],\"PublicKeyCredentialDescriptorJson\":[],\"PublicKeyCredentialEntity\":[],\"PublicKeyCredentialHints\":[],\"PublicKeyCredentialParameters\":[],\"PublicKeyCredentialRequestOptions\":[],\"PublicKeyCredentialRequestOptionsJson\":[],\"PublicKeyCredentialRpEntity\":[],\"PublicKeyCredentialType\":[],\"PublicKeyCredentialUserEntity\":[],\"PublicKeyCredentialUserEntityJson\":[],\"PushEncryptionKeyName\":[],\"PushEvent\":[\"Event\",\"ExtendableEvent\"],\"PushEventInit\":[],\"PushManager\":[],\"PushMessageData\":[],\"PushPermissionState\":[],\"PushSubscription\":[],\"PushSubscriptionInit\":[],\"PushSubscriptionJson\":[],\"PushSubscriptionKeys\":[],\"PushSubscriptionOptions\":[],\"PushSubscriptionOptionsInit\":[],\"QueryOptions\":[],\"QueuingStrategy\":[],\"QueuingStrategyInit\":[],\"RadioNodeList\":[\"NodeList\"],\"Range\":[],\"RcwnPerfStats\":[],\"RcwnStatus\":[],\"ReadableByteStreamController\":[],\"ReadableStream\":[],\"ReadableStreamByobReader\":[],\"ReadableStreamByobRequest\":[],\"ReadableStreamDefaultController\":[],\"ReadableStreamDefaultReader\":[],\"ReadableStreamGetReaderOptions\":[],\"ReadableStreamIteratorOptions\":[],\"ReadableStreamReadResult\":[],\"ReadableStreamReaderMode\":[],\"ReadableStreamType\":[],\"ReadableWritablePair\":[],\"RecordingState\":[],\"ReferrerPolicy\":[],\"RegisterRequest\":[],\"RegisterResponse\":[],\"RegisteredKey\":[],\"RegistrationOptions\":[],\"RegistrationResponseJson\":[],\"Request\":[],\"RequestCache\":[],\"RequestCredentials\":[],\"RequestDestination\":[],\"RequestDeviceOptions\":[],\"RequestInit\":[],\"RequestMediaKeySystemAccessNotification\":[],\"RequestMode\":[],\"RequestRedirect\":[],\"ResidentKeyRequirement\":[],\"ResizeObserver\":[],\"ResizeObserverBoxOptions\":[],\"ResizeObserverEntry\":[],\"ResizeObserverOptions\":[],\"ResizeObserverSize\":[],\"ResizeQuality\":[],\"Response\":[],\"ResponseInit\":[],\"ResponseType\":[],\"RsaHashedImportParams\":[],\"RsaOaepParams\":[],\"RsaOtherPrimesInfo\":[],\"RsaPssParams\":[],\"RtcAnswerOptions\":[],\"RtcBundlePolicy\":[],\"RtcCertificate\":[],\"RtcCertificateExpiration\":[],\"RtcCodecStats\":[],\"RtcConfiguration\":[],\"RtcDataChannel\":[\"EventTarget\"],\"RtcDataChannelEvent\":[\"Event\"],\"RtcDataChannelEventInit\":[],\"RtcDataChannelInit\":[],\"RtcDataChannelState\":[],\"RtcDataChannelType\":[],\"RtcDegradationPreference\":[],\"RtcEncodedAudioFrame\":[],\"RtcEncodedAudioFrameMetadata\":[],\"RtcEncodedAudioFrameOptions\":[],\"RtcEncodedVideoFrame\":[],\"RtcEncodedVideoFrameMetadata\":[],\"RtcEncodedVideoFrameOptions\":[],\"RtcEncodedVideoFrameType\":[],\"RtcFecParameters\":[],\"RtcIceCandidate\":[],\"RtcIceCandidateInit\":[],\"RtcIceCandidatePairStats\":[],\"RtcIceCandidateStats\":[],\"RtcIceComponentStats\":[],\"RtcIceConnectionState\":[],\"RtcIceCredentialType\":[],\"RtcIceGatheringState\":[],\"RtcIceServer\":[],\"RtcIceTransportPolicy\":[],\"RtcIdentityAssertion\":[],\"RtcIdentityAssertionResult\":[],\"RtcIdentityProvider\":[],\"RtcIdentityProviderDetails\":[],\"RtcIdentityProviderOptions\":[],\"RtcIdentityProviderRegistrar\":[],\"RtcIdentityValidationResult\":[],\"RtcInboundRtpStreamStats\":[],\"RtcMediaStreamStats\":[],\"RtcMediaStreamTrackStats\":[],\"RtcOfferAnswerOptions\":[],\"RtcOfferOptions\":[],\"RtcOutboundRtpStreamStats\":[],\"RtcPeerConnection\":[\"EventTarget\"],\"RtcPeerConnectionIceErrorEvent\":[\"Event\"],\"RtcPeerConnectionIceEvent\":[\"Event\"],\"RtcPeerConnectionIceEventInit\":[],\"RtcPeerConnectionState\":[],\"RtcPriorityType\":[],\"RtcRtcpParameters\":[],\"RtcRtpCapabilities\":[],\"RtcRtpCodecCapability\":[],\"RtcRtpCodecParameters\":[],\"RtcRtpContributingSource\":[],\"RtcRtpEncodingParameters\":[],\"RtcRtpHeaderExtensionCapability\":[],\"RtcRtpHeaderExtensionParameters\":[],\"RtcRtpParameters\":[],\"RtcRtpReceiver\":[],\"RtcRtpScriptTransform\":[],\"RtcRtpScriptTransformer\":[\"EventTarget\"],\"RtcRtpSender\":[],\"RtcRtpSourceEntry\":[],\"RtcRtpSourceEntryType\":[],\"RtcRtpSynchronizationSource\":[],\"RtcRtpTransceiver\":[],\"RtcRtpTransceiverDirection\":[],\"RtcRtpTransceiverInit\":[],\"RtcRtxParameters\":[],\"RtcSdpType\":[],\"RtcSessionDescription\":[],\"RtcSessionDescriptionInit\":[],\"RtcSignalingState\":[],\"RtcStats\":[],\"RtcStatsIceCandidatePairState\":[],\"RtcStatsIceCandidateType\":[],\"RtcStatsReport\":[],\"RtcStatsReportInternal\":[],\"RtcStatsType\":[],\"RtcTrackEvent\":[\"Event\"],\"RtcTrackEventInit\":[],\"RtcTransformEvent\":[\"Event\"],\"RtcTransportStats\":[],\"RtcdtmfSender\":[\"EventTarget\"],\"RtcdtmfToneChangeEvent\":[\"Event\"],\"RtcdtmfToneChangeEventInit\":[],\"RtcrtpContributingSourceStats\":[],\"RtcrtpStreamStats\":[],\"SFrameTransform\":[\"EventTarget\"],\"SFrameTransformErrorEvent\":[\"Event\"],\"SFrameTransformErrorEventInit\":[],\"SFrameTransformErrorEventType\":[],\"SFrameTransformOptions\":[],\"SFrameTransformRole\":[],\"SaveFilePickerOptions\":[],\"Scheduler\":[],\"SchedulerPostTaskOptions\":[],\"Scheduling\":[],\"Screen\":[\"EventTarget\"],\"ScreenColorGamut\":[],\"ScreenLuminance\":[],\"ScreenOrientation\":[\"EventTarget\"],\"ScriptProcessorNode\":[\"AudioNode\",\"EventTarget\"],\"ScrollAreaEvent\":[\"Event\",\"UiEvent\"],\"ScrollBehavior\":[],\"ScrollBoxObject\":[],\"ScrollIntoViewOptions\":[],\"ScrollLogicalPosition\":[],\"ScrollOptions\":[],\"ScrollRestoration\":[],\"ScrollSetting\":[],\"ScrollState\":[],\"ScrollToOptions\":[],\"ScrollViewChangeEventInit\":[],\"SecurityPolicyViolationEvent\":[\"Event\"],\"SecurityPolicyViolationEventDisposition\":[],\"SecurityPolicyViolationEventInit\":[],\"Selection\":[],\"SelectionMode\":[],\"Serial\":[\"EventTarget\"],\"SerialInputSignals\":[],\"SerialOptions\":[],\"SerialOutputSignals\":[],\"SerialPort\":[\"EventTarget\"],\"SerialPortFilter\":[],\"SerialPortInfo\":[],\"SerialPortRequestOptions\":[],\"ServerSocketOptions\":[],\"ServiceWorker\":[\"EventTarget\"],\"ServiceWorkerContainer\":[\"EventTarget\"],\"ServiceWorkerGlobalScope\":[\"EventTarget\",\"WorkerGlobalScope\"],\"ServiceWorkerRegistration\":[\"EventTarget\"],\"ServiceWorkerState\":[],\"ServiceWorkerUpdateViaCache\":[],\"ShadowRoot\":[\"DocumentFragment\",\"EventTarget\",\"Node\"],\"ShadowRootInit\":[],\"ShadowRootMode\":[],\"ShareData\":[],\"SharedWorker\":[\"EventTarget\"],\"SharedWorkerGlobalScope\":[\"EventTarget\",\"WorkerGlobalScope\"],\"SignResponse\":[],\"SocketElement\":[],\"SocketOptions\":[],\"SocketReadyState\":[],\"SocketsDict\":[],\"SourceBuffer\":[\"EventTarget\"],\"SourceBufferAppendMode\":[],\"SourceBufferList\":[\"EventTarget\"],\"SpeechGrammar\":[],\"SpeechGrammarList\":[],\"SpeechRecognition\":[\"EventTarget\"],\"SpeechRecognitionAlternative\":[],\"SpeechRecognitionError\":[\"Event\"],\"SpeechRecognitionErrorCode\":[],\"SpeechRecognitionErrorInit\":[],\"SpeechRecognitionEvent\":[\"Event\"],\"SpeechRecognitionEventInit\":[],\"SpeechRecognitionResult\":[],\"SpeechRecognitionResultList\":[],\"SpeechSynthesis\":[\"EventTarget\"],\"SpeechSynthesisErrorCode\":[],\"SpeechSynthesisErrorEvent\":[\"Event\",\"SpeechSynthesisEvent\"],\"SpeechSynthesisErrorEventInit\":[],\"SpeechSynthesisEvent\":[\"Event\"],\"SpeechSynthesisEventInit\":[],\"SpeechSynthesisUtterance\":[\"EventTarget\"],\"SpeechSynthesisVoice\":[],\"StereoPannerNode\":[\"AudioNode\",\"EventTarget\"],\"StereoPannerOptions\":[],\"Storage\":[],\"StorageEstimate\":[],\"StorageEvent\":[\"Event\"],\"StorageEventInit\":[],\"StorageManager\":[],\"StorageType\":[],\"StreamPipeOptions\":[],\"StyleRuleChangeEventInit\":[],\"StyleSheet\":[],\"StyleSheetApplicableStateChangeEventInit\":[],\"StyleSheetChangeEventInit\":[],\"StyleSheetList\":[],\"SubmitEvent\":[\"Event\"],\"SubmitEventInit\":[],\"SubtleCrypto\":[],\"SupportedType\":[],\"SvcOutputMetadata\":[],\"SvgAngle\":[],\"SvgAnimateElement\":[\"Element\",\"EventTarget\",\"Node\",\"SvgAnimationElement\",\"SvgElement\"],\"SvgAnimateMotionElement\":[\"Element\",\"EventTarget\",\"Node\",\"SvgAnimationElement\",\"SvgElement\"],\"SvgAnimateTransformElement\":[\"Element\",\"EventTarget\",\"Node\",\"SvgAnimationElement\",\"SvgElement\"],\"SvgAnimatedAngle\":[],\"SvgAnimatedBoolean\":[],\"SvgAnimatedEnumeration\":[],\"SvgAnimatedInteger\":[],\"SvgAnimatedLength\":[],\"SvgAnimatedLengthList\":[],\"SvgAnimatedNumber\":[],\"SvgAnimatedNumberList\":[],\"SvgAnimatedPreserveAspectRatio\":[],\"SvgAnimatedRect\":[],\"SvgAnimatedString\":[],\"SvgAnimatedTransformList\":[],\"SvgAnimationElement\":[\"Element\",\"EventTarget\",\"Node\",\"SvgElement\"],\"SvgBoundingBoxOptions\":[],\"SvgCircleElement\":[\"Element\",\"EventTarget\",\"Node\",\"SvgElement\",\"SvgGeometryElement\",\"SvgGraphicsElement\"],\"SvgClipPathElement\":[\"Element\",\"EventTarget\",\"Node\",\"SvgElement\"],\"SvgComponentTransferFunctionElement\":[\"Element\",\"EventTarget\",\"Node\",\"SvgElement\"],\"SvgDefsElement\":[\"Element\",\"EventTarget\",\"Node\",\"SvgElement\",\"SvgGraphicsElement\"],\"SvgDescElement\":[\"Element\",\"EventTarget\",\"Node\",\"SvgElement\"],\"SvgElement\":[\"Element\",\"EventTarget\",\"Node\"],\"SvgEllipseElement\":[\"Element\",\"EventTarget\",\"Node\",\"SvgElement\",\"SvgGeometryElement\",\"SvgGraphicsElement\"],\"SvgFilterElement\":[\"Element\",\"EventTarget\",\"Node\",\"SvgElement\"],\"SvgForeignObjectElement\":[\"Element\",\"EventTarget\",\"Node\",\"SvgElement\",\"SvgGraphicsElement\"],\"SvgGeometryElement\":[\"Element\",\"EventTarget\",\"Node\",\"SvgElement\",\"SvgGraphicsElement\"],\"SvgGradientElement\":[\"Element\",\"EventTarget\",\"Node\",\"SvgElement\"],\"SvgGraphicsElement\":[\"Element\",\"EventTarget\",\"Node\",\"SvgElement\"],\"SvgImageElement\":[\"Element\",\"EventTarget\",\"Node\",\"SvgElement\",\"SvgGraphicsElement\"],\"SvgLength\":[],\"SvgLengthList\":[],\"SvgLineElement\":[\"Element\",\"EventTarget\",\"Node\",\"SvgElement\",\"SvgGeometryElement\",\"SvgGraphicsElement\"],\"SvgLinearGradientElement\":[\"Element\",\"EventTarget\",\"Node\",\"SvgElement\",\"SvgGradientElement\"],\"SvgMarkerElement\":[\"Element\",\"EventTarget\",\"Node\",\"SvgElement\"],\"SvgMaskElement\":[\"Element\",\"EventTarget\",\"Node\",\"SvgElement\"],\"SvgMatrix\":[],\"SvgMetadataElement\":[\"Element\",\"EventTarget\",\"Node\",\"SvgElement\"],\"SvgNumber\":[],\"SvgNumberList\":[],\"SvgPathElement\":[\"Element\",\"EventTarget\",\"Node\",\"SvgElement\",\"SvgGeometryElement\",\"SvgGraphicsElement\"],\"SvgPathSeg\":[],\"SvgPathSegArcAbs\":[\"SvgPathSeg\"],\"SvgPathSegArcRel\":[\"SvgPathSeg\"],\"SvgPathSegClosePath\":[\"SvgPathSeg\"],\"SvgPathSegCurvetoCubicAbs\":[\"SvgPathSeg\"],\"SvgPathSegCurvetoCubicRel\":[\"SvgPathSeg\"],\"SvgPathSegCurvetoCubicSmoothAbs\":[\"SvgPathSeg\"],\"SvgPathSegCurvetoCubicSmoothRel\":[\"SvgPathSeg\"],\"SvgPathSegCurvetoQuadraticAbs\":[\"SvgPathSeg\"],\"SvgPathSegCurvetoQuadraticRel\":[\"SvgPathSeg\"],\"SvgPathSegCurvetoQuadraticSmoothAbs\":[\"SvgPathSeg\"],\"SvgPathSegCurvetoQuadraticSmoothRel\":[\"SvgPathSeg\"],\"SvgPathSegLinetoAbs\":[\"SvgPathSeg\"],\"SvgPathSegLinetoHorizontalAbs\":[\"SvgPathSeg\"],\"SvgPathSegLinetoHorizontalRel\":[\"SvgPathSeg\"],\"SvgPathSegLinetoRel\":[\"SvgPathSeg\"],\"SvgPathSegLinetoVerticalAbs\":[\"SvgPathSeg\"],\"SvgPathSegLinetoVerticalRel\":[\"SvgPathSeg\"],\"SvgPathSegList\":[],\"SvgPathSegMovetoAbs\":[\"SvgPathSeg\"],\"SvgPathSegMovetoRel\":[\"SvgPathSeg\"],\"SvgPatternElement\":[\"Element\",\"EventTarget\",\"Node\",\"SvgElement\"],\"SvgPoint\":[],\"SvgPointList\":[],\"SvgPolygonElement\":[\"Element\",\"EventTarget\",\"Node\",\"SvgElement\",\"SvgGeometryElement\",\"SvgGraphicsElement\"],\"SvgPolylineElement\":[\"Element\",\"EventTarget\",\"Node\",\"SvgElement\",\"SvgGeometryElement\",\"SvgGraphicsElement\"],\"SvgPreserveAspectRatio\":[],\"SvgRadialGradientElement\":[\"Element\",\"EventTarget\",\"Node\",\"SvgElement\",\"SvgGradientElement\"],\"SvgRect\":[],\"SvgRectElement\":[\"Element\",\"EventTarget\",\"Node\",\"SvgElement\",\"SvgGeometryElement\",\"SvgGraphicsElement\"],\"SvgScriptElement\":[\"Element\",\"EventTarget\",\"Node\",\"SvgElement\"],\"SvgSetElement\":[\"Element\",\"EventTarget\",\"Node\",\"SvgAnimationElement\",\"SvgElement\"],\"SvgStopElement\":[\"Element\",\"EventTarget\",\"Node\",\"SvgElement\"],\"SvgStringList\":[],\"SvgStyleElement\":[\"Element\",\"EventTarget\",\"Node\",\"SvgElement\"],\"SvgSwitchElement\":[\"Element\",\"EventTarget\",\"Node\",\"SvgElement\",\"SvgGraphicsElement\"],\"SvgSymbolElement\":[\"Element\",\"EventTarget\",\"Node\",\"SvgElement\"],\"SvgTextContentElement\":[\"Element\",\"EventTarget\",\"Node\",\"SvgElement\",\"SvgGraphicsElement\"],\"SvgTextElement\":[\"Element\",\"EventTarget\",\"Node\",\"SvgElement\",\"SvgGraphicsElement\",\"SvgTextContentElement\",\"SvgTextPositioningElement\"],\"SvgTextPathElement\":[\"Element\",\"EventTarget\",\"Node\",\"SvgElement\",\"SvgGraphicsElement\",\"SvgTextContentElement\"],\"SvgTextPositioningElement\":[\"Element\",\"EventTarget\",\"Node\",\"SvgElement\",\"SvgGraphicsElement\",\"SvgTextContentElement\"],\"SvgTitleElement\":[\"Element\",\"EventTarget\",\"Node\",\"SvgElement\"],\"SvgTransform\":[],\"SvgTransformList\":[],\"SvgUnitTypes\":[],\"SvgUseElement\":[\"Element\",\"EventTarget\",\"Node\",\"SvgElement\",\"SvgGraphicsElement\"],\"SvgViewElement\":[\"Element\",\"EventTarget\",\"Node\",\"SvgElement\"],\"SvgZoomAndPan\":[],\"SvgaElement\":[\"Element\",\"EventTarget\",\"Node\",\"SvgElement\",\"SvgGraphicsElement\"],\"SvgfeBlendElement\":[\"Element\",\"EventTarget\",\"Node\",\"SvgElement\"],\"SvgfeColorMatrixElement\":[\"Element\",\"EventTarget\",\"Node\",\"SvgElement\"],\"SvgfeComponentTransferElement\":[\"Element\",\"EventTarget\",\"Node\",\"SvgElement\"],\"SvgfeCompositeElement\":[\"Element\",\"EventTarget\",\"Node\",\"SvgElement\"],\"SvgfeConvolveMatrixElement\":[\"Element\",\"EventTarget\",\"Node\",\"SvgElement\"],\"SvgfeDiffuseLightingElement\":[\"Element\",\"EventTarget\",\"Node\",\"SvgElement\"],\"SvgfeDisplacementMapElement\":[\"Element\",\"EventTarget\",\"Node\",\"SvgElement\"],\"SvgfeDistantLightElement\":[\"Element\",\"EventTarget\",\"Node\",\"SvgElement\"],\"SvgfeDropShadowElement\":[\"Element\",\"EventTarget\",\"Node\",\"SvgElement\"],\"SvgfeFloodElement\":[\"Element\",\"EventTarget\",\"Node\",\"SvgElement\"],\"SvgfeFuncAElement\":[\"Element\",\"EventTarget\",\"Node\",\"SvgComponentTransferFunctionElement\",\"SvgElement\"],\"SvgfeFuncBElement\":[\"Element\",\"EventTarget\",\"Node\",\"SvgComponentTransferFunctionElement\",\"SvgElement\"],\"SvgfeFuncGElement\":[\"Element\",\"EventTarget\",\"Node\",\"SvgComponentTransferFunctionElement\",\"SvgElement\"],\"SvgfeFuncRElement\":[\"Element\",\"EventTarget\",\"Node\",\"SvgComponentTransferFunctionElement\",\"SvgElement\"],\"SvgfeGaussianBlurElement\":[\"Element\",\"EventTarget\",\"Node\",\"SvgElement\"],\"SvgfeImageElement\":[\"Element\",\"EventTarget\",\"Node\",\"SvgElement\"],\"SvgfeMergeElement\":[\"Element\",\"EventTarget\",\"Node\",\"SvgElement\"],\"SvgfeMergeNodeElement\":[\"Element\",\"EventTarget\",\"Node\",\"SvgElement\"],\"SvgfeMorphologyElement\":[\"Element\",\"EventTarget\",\"Node\",\"SvgElement\"],\"SvgfeOffsetElement\":[\"Element\",\"EventTarget\",\"Node\",\"SvgElement\"],\"SvgfePointLightElement\":[\"Element\",\"EventTarget\",\"Node\",\"SvgElement\"],\"SvgfeSpecularLightingElement\":[\"Element\",\"EventTarget\",\"Node\",\"SvgElement\"],\"SvgfeSpotLightElement\":[\"Element\",\"EventTarget\",\"Node\",\"SvgElement\"],\"SvgfeTileElement\":[\"Element\",\"EventTarget\",\"Node\",\"SvgElement\"],\"SvgfeTurbulenceElement\":[\"Element\",\"EventTarget\",\"Node\",\"SvgElement\"],\"SvggElement\":[\"Element\",\"EventTarget\",\"Node\",\"SvgElement\",\"SvgGraphicsElement\"],\"SvgmPathElement\":[\"Element\",\"EventTarget\",\"Node\",\"SvgElement\"],\"SvgsvgElement\":[\"Element\",\"EventTarget\",\"Node\",\"SvgElement\",\"SvgGraphicsElement\"],\"SvgtSpanElement\":[\"Element\",\"EventTarget\",\"Node\",\"SvgElement\",\"SvgGraphicsElement\",\"SvgTextContentElement\",\"SvgTextPositioningElement\"],\"TaskController\":[\"AbortController\"],\"TaskControllerInit\":[],\"TaskPriority\":[],\"TaskPriorityChangeEvent\":[\"Event\"],\"TaskPriorityChangeEventInit\":[],\"TaskSignal\":[\"AbortSignal\",\"EventTarget\"],\"TaskSignalAnyInit\":[],\"TcpReadyState\":[],\"TcpServerSocket\":[\"EventTarget\"],\"TcpServerSocketEvent\":[\"Event\"],\"TcpServerSocketEventInit\":[],\"TcpSocket\":[\"EventTarget\"],\"TcpSocketBinaryType\":[],\"TcpSocketErrorEvent\":[\"Event\"],\"TcpSocketErrorEventInit\":[],\"TcpSocketEvent\":[\"Event\"],\"TcpSocketEventInit\":[],\"Text\":[\"CharacterData\",\"EventTarget\",\"Node\"],\"TextDecodeOptions\":[],\"TextDecoder\":[],\"TextDecoderOptions\":[],\"TextEncoder\":[],\"TextMetrics\":[],\"TextTrack\":[\"EventTarget\"],\"TextTrackCue\":[\"EventTarget\"],\"TextTrackCueList\":[],\"TextTrackKind\":[],\"TextTrackList\":[\"EventTarget\"],\"TextTrackMode\":[],\"TimeEvent\":[\"Event\"],\"TimeRanges\":[],\"ToggleEvent\":[\"Event\"],\"ToggleEventInit\":[],\"TokenBinding\":[],\"TokenBindingStatus\":[],\"Touch\":[],\"TouchEvent\":[\"Event\",\"UiEvent\"],\"TouchEventInit\":[],\"TouchInit\":[],\"TouchList\":[],\"TrackEvent\":[\"Event\"],\"TrackEventInit\":[],\"TransformStream\":[],\"TransformStreamDefaultController\":[],\"Transformer\":[],\"TransitionEvent\":[\"Event\"],\"TransitionEventInit\":[],\"Transport\":[],\"TreeBoxObject\":[],\"TreeCellInfo\":[],\"TreeView\":[],\"TreeWalker\":[],\"U2f\":[],\"U2fClientData\":[],\"ULongRange\":[],\"UaDataValues\":[],\"UaLowEntropyJson\":[],\"UdpMessageEventInit\":[],\"UdpOptions\":[],\"UiEvent\":[\"Event\"],\"UiEventInit\":[],\"UnderlyingSink\":[],\"UnderlyingSource\":[],\"Url\":[],\"UrlSearchParams\":[],\"Usb\":[\"EventTarget\"],\"UsbAlternateInterface\":[],\"UsbConfiguration\":[],\"UsbConnectionEvent\":[\"Event\"],\"UsbConnectionEventInit\":[],\"UsbControlTransferParameters\":[],\"UsbDevice\":[],\"UsbDeviceFilter\":[],\"UsbDeviceRequestOptions\":[],\"UsbDirection\":[],\"UsbEndpoint\":[],\"UsbEndpointType\":[],\"UsbInTransferResult\":[],\"UsbInterface\":[],\"UsbIsochronousInTransferPacket\":[],\"UsbIsochronousInTransferResult\":[],\"UsbIsochronousOutTransferPacket\":[],\"UsbIsochronousOutTransferResult\":[],\"UsbOutTransferResult\":[],\"UsbPermissionDescriptor\":[],\"UsbPermissionResult\":[\"EventTarget\",\"PermissionStatus\"],\"UsbPermissionStorage\":[],\"UsbRecipient\":[],\"UsbRequestType\":[],\"UsbTransferStatus\":[],\"UserActivation\":[],\"UserProximityEvent\":[\"Event\"],\"UserProximityEventInit\":[],\"UserVerificationRequirement\":[],\"ValidityState\":[],\"ValueEvent\":[\"Event\"],\"ValueEventInit\":[],\"VideoColorPrimaries\":[],\"VideoColorSpace\":[],\"VideoColorSpaceInit\":[],\"VideoConfiguration\":[],\"VideoDecoder\":[],\"VideoDecoderConfig\":[],\"VideoDecoderInit\":[],\"VideoDecoderSupport\":[],\"VideoEncoder\":[],\"VideoEncoderConfig\":[],\"VideoEncoderEncodeOptions\":[],\"VideoEncoderInit\":[],\"VideoEncoderSupport\":[],\"VideoFacingModeEnum\":[],\"VideoFrame\":[],\"VideoFrameBufferInit\":[],\"VideoFrameCopyToOptions\":[],\"VideoFrameInit\":[],\"VideoMatrixCoefficients\":[],\"VideoPixelFormat\":[],\"VideoPlaybackQuality\":[],\"VideoStreamTrack\":[\"EventTarget\",\"MediaStreamTrack\"],\"VideoTrack\":[],\"VideoTrackList\":[\"EventTarget\"],\"VideoTransferCharacteristics\":[],\"ViewTransition\":[],\"VisibilityState\":[],\"VisualViewport\":[\"EventTarget\"],\"VoidCallback\":[],\"VrDisplay\":[\"EventTarget\"],\"VrDisplayCapabilities\":[],\"VrEye\":[],\"VrEyeParameters\":[],\"VrFieldOfView\":[],\"VrFrameData\":[],\"VrLayer\":[],\"VrMockController\":[],\"VrMockDisplay\":[],\"VrPose\":[],\"VrServiceTest\":[],\"VrStageParameters\":[],\"VrSubmitFrameResult\":[],\"VttCue\":[\"EventTarget\",\"TextTrackCue\"],\"VttRegion\":[],\"WakeLock\":[],\"WakeLockSentinel\":[\"EventTarget\"],\"WakeLockType\":[],\"WatchAdvertisementsOptions\":[],\"WaveShaperNode\":[\"AudioNode\",\"EventTarget\"],\"WaveShaperOptions\":[],\"WebGl2RenderingContext\":[],\"WebGlActiveInfo\":[],\"WebGlBuffer\":[],\"WebGlContextAttributes\":[],\"WebGlContextEvent\":[\"Event\"],\"WebGlContextEventInit\":[],\"WebGlFramebuffer\":[],\"WebGlPowerPreference\":[],\"WebGlProgram\":[],\"WebGlQuery\":[],\"WebGlRenderbuffer\":[],\"WebGlRenderingContext\":[],\"WebGlSampler\":[],\"WebGlShader\":[],\"WebGlShaderPrecisionFormat\":[],\"WebGlSync\":[],\"WebGlTexture\":[],\"WebGlTransformFeedback\":[],\"WebGlUniformLocation\":[],\"WebGlVertexArrayObject\":[],\"WebKitCssMatrix\":[\"DomMatrix\",\"DomMatrixReadOnly\"],\"WebSocket\":[\"EventTarget\"],\"WebSocketDict\":[],\"WebSocketElement\":[],\"WebTransport\":[],\"WebTransportBidirectionalStream\":[],\"WebTransportCloseInfo\":[],\"WebTransportCongestionControl\":[],\"WebTransportDatagramDuplexStream\":[],\"WebTransportDatagramStats\":[],\"WebTransportError\":[\"DomException\"],\"WebTransportErrorOptions\":[],\"WebTransportErrorSource\":[],\"WebTransportHash\":[],\"WebTransportOptions\":[],\"WebTransportReceiveStream\":[\"ReadableStream\"],\"WebTransportReceiveStreamStats\":[],\"WebTransportReliabilityMode\":[],\"WebTransportSendStream\":[\"WritableStream\"],\"WebTransportSendStreamOptions\":[],\"WebTransportSendStreamStats\":[],\"WebTransportStats\":[],\"WebglColorBufferFloat\":[],\"WebglCompressedTextureAstc\":[],\"WebglCompressedTextureAtc\":[],\"WebglCompressedTextureEtc\":[],\"WebglCompressedTextureEtc1\":[],\"WebglCompressedTexturePvrtc\":[],\"WebglCompressedTextureS3tc\":[],\"WebglCompressedTextureS3tcSrgb\":[],\"WebglDebugRendererInfo\":[],\"WebglDebugShaders\":[],\"WebglDepthTexture\":[],\"WebglDrawBuffers\":[],\"WebglLoseContext\":[],\"WebglMultiDraw\":[],\"WellKnownDirectory\":[],\"WgslLanguageFeatures\":[],\"WheelEvent\":[\"Event\",\"MouseEvent\",\"UiEvent\"],\"WheelEventInit\":[],\"WidevineCdmManifest\":[],\"Window\":[\"EventTarget\"],\"WindowClient\":[\"Client\"],\"Worker\":[\"EventTarget\"],\"WorkerDebuggerGlobalScope\":[\"EventTarget\"],\"WorkerGlobalScope\":[\"EventTarget\"],\"WorkerLocation\":[],\"WorkerNavigator\":[],\"WorkerOptions\":[],\"WorkerType\":[],\"Worklet\":[],\"WorkletGlobalScope\":[],\"WorkletOptions\":[],\"WritableStream\":[],\"WritableStreamDefaultController\":[],\"WritableStreamDefaultWriter\":[],\"WriteCommandType\":[],\"WriteParams\":[],\"XPathExpression\":[],\"XPathNsResolver\":[],\"XPathResult\":[],\"XmlDocument\":[\"Document\",\"EventTarget\",\"Node\"],\"XmlHttpRequest\":[\"EventTarget\",\"XmlHttpRequestEventTarget\"],\"XmlHttpRequestEventTarget\":[\"EventTarget\"],\"XmlHttpRequestResponseType\":[],\"XmlHttpRequestUpload\":[\"EventTarget\",\"XmlHttpRequestEventTarget\"],\"XmlSerializer\":[],\"XrBoundedReferenceSpace\":[\"EventTarget\",\"XrReferenceSpace\",\"XrSpace\"],\"XrEye\":[],\"XrFrame\":[],\"XrHand\":[],\"XrHandJoint\":[],\"XrHandedness\":[],\"XrInputSource\":[],\"XrInputSourceArray\":[],\"XrInputSourceEvent\":[\"Event\"],\"XrInputSourceEventInit\":[],\"XrInputSourcesChangeEvent\":[\"Event\"],\"XrInputSourcesChangeEventInit\":[],\"XrJointPose\":[\"XrPose\"],\"XrJointSpace\":[\"EventTarget\",\"XrSpace\"],\"XrLayer\":[\"EventTarget\"],\"XrPermissionDescriptor\":[],\"XrPermissionStatus\":[\"EventTarget\",\"PermissionStatus\"],\"XrPose\":[],\"XrReferenceSpace\":[\"EventTarget\",\"XrSpace\"],\"XrReferenceSpaceEvent\":[\"Event\"],\"XrReferenceSpaceEventInit\":[],\"XrReferenceSpaceType\":[],\"XrRenderState\":[],\"XrRenderStateInit\":[],\"XrRigidTransform\":[],\"XrSession\":[\"EventTarget\"],\"XrSessionEvent\":[\"Event\"],\"XrSessionEventInit\":[],\"XrSessionInit\":[],\"XrSessionMode\":[],\"XrSessionSupportedPermissionDescriptor\":[],\"XrSpace\":[\"EventTarget\"],\"XrSystem\":[\"EventTarget\"],\"XrTargetRayMode\":[],\"XrView\":[],\"XrViewerPose\":[\"XrPose\"],\"XrViewport\":[],\"XrVisibilityState\":[],\"XrWebGlLayer\":[\"EventTarget\",\"XrLayer\"],\"XrWebGlLayerInit\":[],\"XsltProcessor\":[],\"console\":[],\"css\":[],\"default\":[\"std\"],\"gpu_buffer_usage\":[],\"gpu_color_write\":[],\"gpu_map_mode\":[],\"gpu_shader_stage\":[],\"gpu_texture_usage\":[],\"std\":[\"wasm-bindgen/std\",\"js-sys/std\"]}}", "web-time_1.1.0": "{\"dependencies\":[{\"default_features\":false,\"features\":[\"alloc\"],\"kind\":\"dev\",\"name\":\"futures-channel\",\"req\":\"^0.3\",\"target\":\"cfg(all(target_family = \\\"wasm\\\", target_feature = \\\"atomics\\\"))\"},{\"default_features\":false,\"kind\":\"dev\",\"name\":\"futures-util\",\"req\":\"^0.3\",\"target\":\"cfg(all(target_family = \\\"wasm\\\", target_feature = \\\"atomics\\\"))\"},{\"features\":[\"js\"],\"kind\":\"dev\",\"name\":\"getrandom\",\"req\":\"^0.2\",\"target\":\"cfg(target_family = \\\"wasm\\\")\"},{\"name\":\"js-sys\",\"req\":\"^0.3.20\",\"target\":\"cfg(all(target_family = \\\"wasm\\\", target_os = \\\"unknown\\\"))\"},{\"features\":[\"macro\"],\"kind\":\"dev\",\"name\":\"pollster\",\"req\":\"^0.3\",\"target\":\"cfg(not(target_family = \\\"wasm\\\"))\"},{\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.8\",\"target\":\"cfg(target_family = \\\"wasm\\\")\"},{\"name\":\"serde\",\"optional\":true,\"req\":\"^1\",\"target\":\"cfg(all(target_family = \\\"wasm\\\", target_os = \\\"unknown\\\"))\"},{\"kind\":\"dev\",\"name\":\"serde_json\",\"req\":\"^1\",\"target\":\"cfg(target_family = \\\"wasm\\\")\"},{\"kind\":\"dev\",\"name\":\"static_assertions\",\"req\":\"^1\"},{\"default_features\":false,\"name\":\"wasm-bindgen\",\"req\":\"^0.2.70\",\"target\":\"cfg(all(target_family = \\\"wasm\\\", target_os = \\\"unknown\\\"))\"},{\"kind\":\"dev\",\"name\":\"wasm-bindgen-futures\",\"req\":\"^0.4\",\"target\":\"cfg(target_family = \\\"wasm\\\")\"},{\"kind\":\"dev\",\"name\":\"wasm-bindgen-test\",\"req\":\"^0.3\",\"target\":\"cfg(target_family = \\\"wasm\\\")\"},{\"features\":[\"WorkerGlobalScope\"],\"kind\":\"dev\",\"name\":\"web-sys\",\"req\":\"^0.3\",\"target\":\"cfg(all(target_family = \\\"wasm\\\", target_feature = \\\"atomics\\\"))\"},{\"features\":[\"CssStyleDeclaration\",\"Document\",\"Element\",\"HtmlTableElement\",\"HtmlTableRowElement\",\"Performance\",\"Window\"],\"kind\":\"dev\",\"name\":\"web-sys\",\"req\":\"^0.3\",\"target\":\"cfg(target_family = \\\"wasm\\\")\"}],\"features\":{\"serde\":[\"dep:serde\"]}}", "webpki-root-certs_1.0.3": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"hex\",\"req\":\"^0.4.3\"},{\"kind\":\"dev\",\"name\":\"percent-encoding\",\"req\":\"^2.3\"},{\"default_features\":false,\"name\":\"pki-types\",\"package\":\"rustls-pki-types\",\"req\":\"^1.8\"},{\"kind\":\"dev\",\"name\":\"ring\",\"req\":\"^0.17.0\"},{\"features\":[\"macros\",\"rt-multi-thread\"],\"kind\":\"dev\",\"name\":\"tokio\",\"req\":\"^1\"},{\"features\":[\"alloc\"],\"kind\":\"dev\",\"name\":\"webpki\",\"package\":\"rustls-webpki\",\"req\":\"^0.103\"},{\"kind\":\"dev\",\"name\":\"x509-parser\",\"req\":\"^0.17.0\"}],\"features\":{}}", "webpki-roots_0.26.11": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"hex\",\"req\":\"^0.4.3\"},{\"name\":\"parent\",\"package\":\"webpki-roots\",\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"percent-encoding\",\"req\":\"^2.3\"},{\"default_features\":false,\"kind\":\"dev\",\"name\":\"pki-types\",\"package\":\"rustls-pki-types\",\"req\":\"^1.8\"},{\"kind\":\"dev\",\"name\":\"rcgen\",\"req\":\"^0.13\"},{\"kind\":\"dev\",\"name\":\"ring\",\"req\":\"^0.17.0\"},{\"kind\":\"dev\",\"name\":\"rustls\",\"req\":\"^0.23\"},{\"features\":[\"macros\",\"rt-multi-thread\"],\"kind\":\"dev\",\"name\":\"tokio\",\"req\":\"^1\"},{\"features\":[\"alloc\"],\"kind\":\"dev\",\"name\":\"webpki\",\"package\":\"rustls-webpki\",\"req\":\"^0.102\"},{\"kind\":\"dev\",\"name\":\"x509-parser\",\"req\":\"^0.17.0\"},{\"kind\":\"dev\",\"name\":\"yasna\",\"req\":\"^0.5.2\"}],\"features\":{}}", "webpki-roots_1.0.3": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"hex\",\"req\":\"^0.4.3\"},{\"kind\":\"dev\",\"name\":\"percent-encoding\",\"req\":\"^2.3\"},{\"default_features\":false,\"name\":\"pki-types\",\"package\":\"rustls-pki-types\",\"req\":\"^1.8\"},{\"kind\":\"dev\",\"name\":\"rcgen\",\"req\":\"^0.14.3\"},{\"kind\":\"dev\",\"name\":\"ring\",\"req\":\"^0.17.0\"},{\"kind\":\"dev\",\"name\":\"rustls\",\"req\":\"^0.23\"},{\"features\":[\"macros\",\"rt-multi-thread\"],\"kind\":\"dev\",\"name\":\"tokio\",\"req\":\"^1\"},{\"features\":[\"alloc\"],\"kind\":\"dev\",\"name\":\"webpki\",\"package\":\"rustls-webpki\",\"req\":\"^0.103\"},{\"kind\":\"dev\",\"name\":\"x509-parser\",\"req\":\"^0.17.0\"},{\"kind\":\"dev\",\"name\":\"yasna\",\"req\":\"^0.5.2\"}],\"features\":{}}", + "webpki-roots_1.0.7": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"aws-lc-rs\",\"req\":\"^1.15.2\"},{\"kind\":\"dev\",\"name\":\"hex\",\"req\":\"^0.4.3\"},{\"kind\":\"dev\",\"name\":\"percent-encoding\",\"req\":\"^2.3\"},{\"default_features\":false,\"name\":\"pki-types\",\"package\":\"rustls-pki-types\",\"req\":\"^1.8\"},{\"kind\":\"dev\",\"name\":\"rcgen\",\"req\":\"^0.14.3\"},{\"kind\":\"dev\",\"name\":\"rustls\",\"req\":\"^0.23\"},{\"features\":[\"macros\",\"rt-multi-thread\"],\"kind\":\"dev\",\"name\":\"tokio\",\"req\":\"^1\"},{\"features\":[\"alloc\"],\"kind\":\"dev\",\"name\":\"webpki\",\"package\":\"rustls-webpki\",\"req\":\"^0.103\"},{\"kind\":\"dev\",\"name\":\"x509-parser\",\"req\":\"^0.18\"},{\"kind\":\"dev\",\"name\":\"yasna\",\"req\":\"^0.6\"}],\"features\":{}}", "which_8.0.2": "{\"dependencies\":[{\"name\":\"libc\",\"optional\":true,\"req\":\"^0.2\",\"target\":\"cfg(any(unix, target_os = \\\"wasi\\\", target_os = \\\"redox\\\"))\"},{\"name\":\"regex\",\"optional\":true,\"req\":\"^1.10.2\"},{\"kind\":\"dev\",\"name\":\"tempfile\",\"req\":\"^3.9.0\"},{\"default_features\":false,\"name\":\"tracing\",\"optional\":true,\"req\":\"^0.1.40\"}],\"features\":{\"default\":[\"real-sys\"],\"real-sys\":[\"dep:libc\"],\"regex\":[\"dep:regex\"],\"tracing\":[\"dep:tracing\"]}}", "widestring_1.2.1": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"debugger_test\",\"req\":\"^0.1\"},{\"kind\":\"dev\",\"name\":\"debugger_test_parser\",\"req\":\"^0.1\"},{\"features\":[\"Win32_System_Diagnostics_Debug\"],\"kind\":\"dev\",\"name\":\"windows-sys\",\"req\":\"^0.59\"}],\"features\":{\"alloc\":[],\"debugger_visualizer\":[\"alloc\"],\"default\":[\"std\"],\"std\":[\"alloc\"]}}", "winapi-util_0.1.11": "{\"dependencies\":[{\"features\":[\"Win32_Foundation\",\"Win32_Storage_FileSystem\",\"Win32_System_Console\",\"Win32_System_SystemInformation\"],\"name\":\"windows-sys\",\"req\":\">=0.48.0, <=0.61\",\"target\":\"cfg(windows)\"}],\"features\":{}}", - "wincode-derive_0.4.5": "{\"dependencies\":[{\"name\":\"darling\",\"req\":\"^0.21.3\"},{\"name\":\"proc-macro2\",\"req\":\"^1.0.106\"},{\"name\":\"quote\",\"req\":\"^1.0.45\"},{\"features\":[\"visit-mut\",\"visit\"],\"name\":\"syn\",\"req\":\"^2.0.117\"}],\"features\":{}}", - "wincode_0.5.4": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"bincode\",\"req\":\"^1.3.3\"},{\"default_features\":false,\"name\":\"bv\",\"optional\":true,\"req\":\"^0.11.1\"},{\"features\":[\"serde\"],\"kind\":\"dev\",\"name\":\"bv\",\"req\":\"^0.11.1\"},{\"default_features\":false,\"name\":\"bytes\",\"optional\":true,\"req\":\"^1.11.1\"},{\"features\":[\"html_reports\"],\"kind\":\"dev\",\"name\":\"criterion\",\"req\":\"^0.7.0\"},{\"default_features\":false,\"name\":\"ecow\",\"optional\":true,\"req\":\"^0.2.6\"},{\"default_features\":false,\"features\":[\"serde\"],\"kind\":\"dev\",\"name\":\"ecow\",\"req\":\"^0.2.6\"},{\"default_features\":false,\"name\":\"indexmap\",\"optional\":true,\"req\":\"^2.13.1\"},{\"default_features\":false,\"features\":[\"serde\"],\"kind\":\"dev\",\"name\":\"indexmap\",\"req\":\"^2.13.1\"},{\"name\":\"pastey\",\"req\":\"^0.2.1\"},{\"kind\":\"build\",\"name\":\"proc-macro2\",\"req\":\"^1.0.106\"},{\"kind\":\"dev\",\"name\":\"proptest\",\"req\":\"^1.11.0\"},{\"kind\":\"dev\",\"name\":\"proptest-derive\",\"req\":\"^0.8.0\"},{\"kind\":\"build\",\"name\":\"quote\",\"req\":\"^1.0.45\"},{\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.9.2\"},{\"features\":[\"derive\",\"rc\"],\"kind\":\"dev\",\"name\":\"serde\",\"req\":\"^1.0.227\"},{\"default_features\":false,\"features\":[\"const_generics\"],\"name\":\"smallvec\",\"optional\":true,\"req\":\"^1.15.1\"},{\"default_features\":false,\"features\":[\"const_generics\",\"serde\"],\"kind\":\"dev\",\"name\":\"smallvec\",\"req\":\"^1.15.1\"},{\"default_features\":false,\"name\":\"solana-short-vec\",\"optional\":true,\"req\":\"^3.2.0\"},{\"kind\":\"dev\",\"name\":\"solana-short-vec\",\"req\":\"^3.2.0\"},{\"default_features\":false,\"name\":\"thiserror\",\"req\":\"^2.0.18\"},{\"default_features\":false,\"name\":\"uuid\",\"optional\":true,\"req\":\"^1.23.0\"},{\"features\":[\"serde\"],\"kind\":\"dev\",\"name\":\"uuid\",\"req\":\"^1.23.0\"},{\"name\":\"wincode-derive\",\"optional\":true,\"req\":\"^0.4.5\"}],\"features\":{\"alloc\":[],\"bv\":[\"std\",\"dep:bv\"],\"bytes\":[\"alloc\",\"dep:bytes\"],\"bytes-extra-platforms\":[\"bytes\",\"bytes/extra-platforms\"],\"default\":[\"std\"],\"derive\":[\"dep:wincode-derive\"],\"ecow\":[\"alloc\",\"dep:ecow\"],\"indexmap\":[\"alloc\",\"dep:indexmap\"],\"smallvec\":[\"alloc\",\"dep:smallvec\"],\"solana-short-vec\":[\"dep:solana-short-vec\"],\"std\":[\"alloc\",\"thiserror/std\"],\"uuid\":[\"dep:uuid\"],\"uuid-serde-compat\":[\"uuid\"]}}", + "wincode-derive_0.4.6": "{\"dependencies\":[{\"name\":\"darling\",\"req\":\"^0.23.0\"},{\"name\":\"proc-macro2\",\"req\":\"^1.0.106\"},{\"name\":\"quote\",\"req\":\"^1.0.45\"},{\"features\":[\"visit-mut\",\"visit\"],\"name\":\"syn\",\"req\":\"^2.0.117\"}],\"features\":{}}", + "wincode_0.5.5": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"bincode\",\"req\":\"^1.3.3\"},{\"default_features\":false,\"name\":\"bv\",\"optional\":true,\"req\":\"^0.11.1\"},{\"features\":[\"serde\"],\"kind\":\"dev\",\"name\":\"bv\",\"req\":\"^0.11.1\"},{\"default_features\":false,\"name\":\"bytes\",\"optional\":true,\"req\":\"^1.11.1\"},{\"features\":[\"html_reports\"],\"kind\":\"dev\",\"name\":\"criterion\",\"req\":\"^0.7.0\"},{\"default_features\":false,\"name\":\"ecow\",\"optional\":true,\"req\":\"^0.2.6\"},{\"default_features\":false,\"features\":[\"serde\"],\"kind\":\"dev\",\"name\":\"ecow\",\"req\":\"^0.2.6\"},{\"default_features\":false,\"name\":\"indexmap\",\"optional\":true,\"req\":\"^2.13.1\"},{\"default_features\":false,\"features\":[\"serde\"],\"kind\":\"dev\",\"name\":\"indexmap\",\"req\":\"^2.13.1\"},{\"name\":\"pastey\",\"req\":\"^0.2.2\"},{\"kind\":\"build\",\"name\":\"proc-macro2\",\"req\":\"^1.0.106\"},{\"kind\":\"dev\",\"name\":\"proptest\",\"req\":\"^1.11.0\"},{\"kind\":\"dev\",\"name\":\"proptest-derive\",\"req\":\"^0.8.0\"},{\"kind\":\"build\",\"name\":\"quote\",\"req\":\"^1.0.45\"},{\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.9.2\"},{\"features\":[\"derive\",\"rc\"],\"kind\":\"dev\",\"name\":\"serde\",\"req\":\"^1.0.227\"},{\"default_features\":false,\"features\":[\"const_generics\"],\"name\":\"smallvec\",\"optional\":true,\"req\":\"^1.15.1\"},{\"default_features\":false,\"features\":[\"const_generics\",\"serde\"],\"kind\":\"dev\",\"name\":\"smallvec\",\"req\":\"^1.15.1\"},{\"default_features\":false,\"name\":\"solana-short-vec\",\"optional\":true,\"req\":\"^3.2.1\"},{\"kind\":\"dev\",\"name\":\"solana-short-vec\",\"req\":\"^3.2.1\"},{\"default_features\":false,\"name\":\"thiserror\",\"req\":\"^2.0.18\"},{\"default_features\":false,\"name\":\"uuid\",\"optional\":true,\"req\":\"^1.23.1\"},{\"features\":[\"serde\"],\"kind\":\"dev\",\"name\":\"uuid\",\"req\":\"^1.23.1\"},{\"name\":\"wincode-derive\",\"optional\":true,\"req\":\"^0.4.6\"}],\"features\":{\"alloc\":[],\"bv\":[\"std\",\"dep:bv\"],\"bytes\":[\"alloc\",\"dep:bytes\"],\"bytes-extra-platforms\":[\"bytes\",\"bytes/extra-platforms\"],\"default\":[\"std\"],\"derive\":[\"dep:wincode-derive\"],\"ecow\":[\"alloc\",\"dep:ecow\"],\"indexmap\":[\"alloc\",\"dep:indexmap\"],\"smallvec\":[\"alloc\",\"dep:smallvec\"],\"solana-short-vec\":[\"dep:solana-short-vec\"],\"std\":[\"alloc\",\"thiserror/std\"],\"uuid\":[\"dep:uuid\"],\"uuid-serde-compat\":[\"uuid\"]}}", "windows-core_0.62.2": "{\"dependencies\":[{\"default_features\":false,\"name\":\"windows-implement\",\"req\":\"^0.60.2\"},{\"default_features\":false,\"name\":\"windows-interface\",\"req\":\"^0.59.3\"},{\"default_features\":false,\"name\":\"windows-link\",\"req\":\"^0.2.1\"},{\"default_features\":false,\"name\":\"windows-result\",\"req\":\"^0.4.1\"},{\"default_features\":false,\"name\":\"windows-strings\",\"req\":\"^0.5.1\"}],\"features\":{\"default\":[\"std\"],\"std\":[\"windows-result/std\",\"windows-strings/std\"]}}", "windows-implement_0.60.2": "{\"dependencies\":[{\"default_features\":false,\"name\":\"proc-macro2\",\"req\":\"^1.0\"},{\"default_features\":false,\"name\":\"quote\",\"req\":\"^1.0\"},{\"default_features\":false,\"features\":[\"parsing\",\"proc-macro\",\"printing\",\"full\",\"clone-impls\"],\"name\":\"syn\",\"req\":\"^2.0\"}],\"features\":{}}", "windows-interface_0.59.3": "{\"dependencies\":[{\"default_features\":false,\"name\":\"proc-macro2\",\"req\":\"^1.0\"},{\"default_features\":false,\"name\":\"quote\",\"req\":\"^1.0\"},{\"default_features\":false,\"features\":[\"parsing\",\"proc-macro\",\"printing\",\"full\",\"clone-impls\"],\"name\":\"syn\",\"req\":\"^2.0\"}],\"features\":{}}", @@ -1403,7 +1379,9 @@ "yansi_1.0.1": "{\"dependencies\":[{\"name\":\"is-terminal\",\"optional\":true,\"req\":\"^0.4.11\"}],\"features\":{\"_nightly\":[],\"alloc\":[],\"default\":[\"std\"],\"detect-env\":[\"std\"],\"detect-tty\":[\"is-terminal\",\"std\"],\"hyperlink\":[\"std\"],\"std\":[\"alloc\"]}}", "yoke-derive_0.8.2": "{\"dependencies\":[{\"name\":\"proc-macro2\",\"req\":\"^1.0.61\"},{\"name\":\"quote\",\"req\":\"^1.0.44\"},{\"features\":[\"fold\"],\"name\":\"syn\",\"req\":\"^2.0.21\"},{\"name\":\"synstructure\",\"req\":\"^0.13.0\"}],\"features\":{}}", "yoke_0.8.2": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"bincode\",\"req\":\"^1.3.1\"},{\"default_features\":false,\"kind\":\"dev\",\"name\":\"postcard\",\"req\":\"^1.0.3\"},{\"default_features\":false,\"kind\":\"dev\",\"name\":\"serde\",\"req\":\"^1.0.220\"},{\"default_features\":false,\"name\":\"stable_deref_trait\",\"req\":\"^1.2.0\"},{\"default_features\":false,\"name\":\"yoke-derive\",\"optional\":true,\"req\":\"^0.8.2\"},{\"default_features\":false,\"name\":\"zerofrom\",\"optional\":true,\"req\":\"^0.1.6\"}],\"features\":{\"alloc\":[\"stable_deref_trait/alloc\",\"zerofrom/alloc\"],\"default\":[\"alloc\",\"zerofrom\"],\"derive\":[\"dep:yoke-derive\",\"zerofrom/derive\"],\"serde\":[],\"zerofrom\":[\"dep:zerofrom\"]}}", + "zerocopy-derive_0.8.48": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"dissimilar\",\"req\":\"^1.0.9\"},{\"kind\":\"dev\",\"name\":\"prettyplease\",\"req\":\"=0.2.17\"},{\"name\":\"proc-macro2\",\"req\":\"^1.0.1\"},{\"name\":\"quote\",\"req\":\"^1.0.40\"},{\"kind\":\"dev\",\"name\":\"rustversion\",\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"static_assertions\",\"req\":\"^1.1\"},{\"features\":[\"full\"],\"name\":\"syn\",\"req\":\"^2.0.46\"},{\"features\":[\"visit\"],\"kind\":\"dev\",\"name\":\"syn\",\"req\":\"^2.0.46\"}],\"features\":{}}", "zerocopy-derive_0.8.50": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"dissimilar\",\"req\":\"^1.0.9\"},{\"kind\":\"dev\",\"name\":\"prettyplease\",\"req\":\"=0.2.17\"},{\"name\":\"proc-macro2\",\"req\":\"^1.0.1\"},{\"name\":\"quote\",\"req\":\"^1.0.40\"},{\"kind\":\"dev\",\"name\":\"rustversion\",\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"static_assertions\",\"req\":\"^1.1\"},{\"features\":[\"full\"],\"name\":\"syn\",\"req\":\"^2.0.46\"},{\"features\":[\"visit\"],\"kind\":\"dev\",\"name\":\"syn\",\"req\":\"^2.0.46\"}],\"features\":{}}", + "zerocopy_0.8.48": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"elain\",\"req\":\"^0.3.0\"},{\"kind\":\"dev\",\"name\":\"itertools\",\"req\":\"^0.11\"},{\"default_features\":false,\"features\":[\"small_rng\"],\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.8.5\"},{\"kind\":\"dev\",\"name\":\"regex\",\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"rustversion\",\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"static_assertions\",\"req\":\"^1.1\"},{\"name\":\"zerocopy-derive\",\"req\":\"=0.8.48\",\"target\":\"cfg(any())\"},{\"name\":\"zerocopy-derive\",\"optional\":true,\"req\":\"=0.8.48\"},{\"kind\":\"dev\",\"name\":\"zerocopy-derive\",\"req\":\"=0.8.48\"}],\"features\":{\"__internal_use_only_features_that_work_on_stable\":[\"alloc\",\"derive\",\"simd\",\"std\"],\"alloc\":[],\"derive\":[\"zerocopy-derive\"],\"float-nightly\":[],\"simd\":[],\"simd-nightly\":[\"simd\"],\"std\":[\"alloc\"]}}", "zerocopy_0.8.50": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"elain\",\"req\":\"^0.3.0\"},{\"kind\":\"dev\",\"name\":\"itertools\",\"req\":\"^0.11\"},{\"default_features\":false,\"features\":[\"small_rng\"],\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.8.5\"},{\"kind\":\"dev\",\"name\":\"regex\",\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"rustversion\",\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"static_assertions\",\"req\":\"^1.1\"},{\"name\":\"zerocopy-derive\",\"req\":\"=0.8.50\",\"target\":\"cfg(any())\"},{\"name\":\"zerocopy-derive\",\"optional\":true,\"req\":\"=0.8.50\"},{\"kind\":\"dev\",\"name\":\"zerocopy-derive\",\"req\":\"=0.8.50\"}],\"features\":{\"__internal_use_only_features_that_work_on_stable\":[\"alloc\",\"derive\",\"simd\",\"std\"],\"alloc\":[],\"derive\":[\"zerocopy-derive\"],\"float-nightly\":[],\"simd\":[],\"simd-nightly\":[\"simd\"],\"std\":[\"alloc\"]}}", "zerofrom-derive_0.1.7": "{\"dependencies\":[{\"name\":\"proc-macro2\",\"req\":\"^1.0.61\"},{\"name\":\"quote\",\"req\":\"^1.0.44\"},{\"features\":[\"fold\"],\"name\":\"syn\",\"req\":\"^2.0.21\"},{\"name\":\"synstructure\",\"req\":\"^0.13.0\"}],\"features\":{}}", "zerofrom_0.1.8": "{\"dependencies\":[{\"default_features\":false,\"name\":\"zerofrom-derive\",\"optional\":true,\"req\":\"^0.1.6\"}],\"features\":{\"alloc\":[],\"default\":[\"alloc\"],\"derive\":[\"dep:zerofrom-derive\"]}}", @@ -1510,6 +1488,7 @@ "rustc-1.92.0-aarch64-apple-darwin.tar.xz": "15dee753c9217dff4cf45d734b29dc13ce6017d8a55fe34eed75022b39a63ff0", "rustc-1.92.0-aarch64-pc-windows-msvc.tar.xz": "07ba5606143de3bc916455714f67732fca05805591396a8190d5e88b8863eea3", "rustc-1.92.0-aarch64-unknown-linux-gnu.tar.xz": "7c8706fad4c038b5eacab0092e15db54d2b365d5f3323ca046fe987f814e7826", + "rustc-1.92.0-src.tar.xz": "ebee170bfe4c4dfc59521a101de651e5534f4dae889756a5c97ca9ea40d0c307", "rustc-1.92.0-x86_64-apple-darwin.tar.xz": "0facbd5d2742c8e97c53d59c9b5b81db6088cfc285d9ecb99523a50d6765fc5c", "rustc-1.92.0-x86_64-pc-windows-msvc.tar.xz": "6aec5aba5384ce1041538e71ebebf671d82e29f4fe697bc7844626d94b0dbfe6", "rustc-1.92.0-x86_64-unknown-linux-gnu.tar.xz": "78b2dd9c6b1fcd2621fa81c611cf5e2d6950690775038b585c64f364422886e0", diff --git a/README.md b/README.md index 0469a288e..3c6c96a73 100644 --- a/README.md +++ b/README.md @@ -53,7 +53,7 @@ NativeLink is trusted in production environments to reduce costs and developer i - Utilizes remote resources to offload computational burden from local machines - Ensures consistency with a uniform, controlled build environment -NativeLink seamlessly integrates with build tools that use the Remote Execution protocol, such as [Bazel](https://bazel.build), [Buck2](https://buck2.build), [Goma](https://chromium.googlesource.com/infra/goma/client/), and [Siso](https://chromium.googlesource.com/build/+/refs/heads/main/siso/README.md). CMake projects work too via [`recc`](https://buildgrid.gitlab.io/recc). See [Build CMake projects with NativeLink](https://nativelink.com/docs/rbe/cmake-recc). It supports Unix-based operating systems and Windows, ensuring broad compatibility across different development environments. +NativeLink seamlessly integrates with build tools that use the Remote Execution protocol, such as [Bazel](https://bazel.build), [Buck2](https://buck2.build), [Goma](https://chromium.googlesource.com/infra/goma/client/), and [Siso](https://chromium.googlesource.com/build/+/refs/heads/main/siso/README.md). CMake projects work too via [`recc`](https://buildgrid.gitlab.io/recc). See [Build CMake projects with NativeLink](https://docs.nativelink.com/getting-started/other-build-systems/cmake-recc). It supports Unix-based operating systems and Windows, ensuring broad compatibility across different development environments. ## 🚀 Quickstart diff --git a/ci.bazelrc b/ci.bazelrc new file mode 100644 index 000000000..90a314957 --- /dev/null +++ b/ci.bazelrc @@ -0,0 +1,54 @@ +# Copyright 2022 The NativeLink Authors. All rights reserved. +# +# Licensed under the Functional Source License, Version 1.1, Apache 2.0 Future License (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# See LICENSE file for details +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# NativeLink Cloud configuration for this repository's own CI. Everything here +# is opt-in via --config; plain local builds are unaffected. CI activates these +# groups through a generated user.bazelrc (see +# .github/actions/setup-nativelink-cloud), which injects the claim endpoints +# and the --remote_header/--bes_header credentials from repository secrets — +# neither endpoints nor secrets belong in this file. + +# --- Behavior groups (endpoint-free) --- + +# Remote cache + BES for lanes that still execute locally. Safe on any host. +common:nl-cache --remote_timeout=600 +common:nl-cache --remote_retries=5 +common:nl-cache --build_event_publish_all_actions +common:nl-cache --bes_upload_mode=fully_async + +# Main-branch mode: populate the shared cache. Only TraceMachina main-branch +# identities may hold a write-capable key; PR lanes instead get a plain +# --remote_upload_local_results=false line from the setup action. +common:nl-write --remote_upload_local_results=true +common:nl-write --remote_cache_async + +# Remote execution. Mirrors the staging-rbe config proven from a developer +# machine: hermetic LLVM plus downloaded Rust toolchains, so workers need no +# GCC/Rust/Nix. Combine only with an *-exec endpoint group, and only on +# x86_64 Linux hosts — macOS auto-applies conflicting LRE platforms from +# lre.bazelrc, which the setup action guards against. +common:nl-rbe --platforms=@llvm//platforms:linux_x86_64_gnu.2.28 +common:nl-rbe --host_platform=@llvm//platforms:linux_x86_64_gnu.2.28 +common:nl-rbe --extra_execution_platforms=@llvm//platforms:linux_x86_64_gnu.2.28 +common:nl-rbe --extra_toolchains=@rust_toolchains//:all +# The default musl interpreter is dynamically linked against the musl loader, +# which glibc-only worker images lack (genrule tools exit 127). +common:nl-rbe --@rules_python//python/config_settings:py_linux_libc=glibc +common:nl-rbe --jobs=200 +common:nl-rbe --remote_download_minimal +common:nl-rbe --remote_retries=8 +common:nl-rbe --experimental_remote_cache_eviction_retries=5 + +# Endpoints deliberately do not appear in this file: the setup action derives +# them from the claim-base repository secret at job time. diff --git a/clippy.toml b/clippy.toml index 925f27f2b..2e3ed029d 100644 --- a/clippy.toml +++ b/clippy.toml @@ -14,4 +14,83 @@ disallowed-methods = [ { path = "tokio::task::spawn_blocking", reason = "use `nativelink-util::task::spawn_blocking` instead" }, { path = "tokio::task::spawn_local", reason = "use one of the `nativelink-util::task` functions instead" }, ] +# Started from the default at https://doc.rust-lang.org/clippy/lint_configuration.html#doc-valid-idents +doc-valid-idents = [ + "AccessKit", + "BibLaTeX", + "BibTeX", + "CamelCase", + "ClojureScript", + "CoAP", + "CoffeeScript", + "CoreFoundation", + "CoreGraphics", + "CoreText", + "DevOps", + "Direct2D", + "Direct3D", + "DirectWrite", + "DirectX", + "ECMAScript", + "EiB", + "FreeBSD", + "GHz", + "GPLv2", + "GPLv3", + "GiB", + "GitHub", + "GitLab", + "GraphQL", + "IPv4", + "IPv6", + "InfiniBand", + "JavaScript", + "KiB", + "LaTeX", + "MHz", + "MiB", + "MinGW", + "NaN", + "NaNs", + "NetApp", + "NetBSD", + "NixOS", + "OAuth", + "OCaml", + "OpenAL", + "OpenBSD", + "OpenDNS", + "OpenExr", + "OpenGL", + "OpenMP", + "OpenSSH", + "OpenSSL", + "OpenStreetMap", + "OpenTelemetry", + "OpenType", + "PiB", + "PostScript", + "PowerPC", + "PowerShell", + "PureScript", + "RoCE", + "THz", + "TeX", + "TensorFlow", + "TiB", + "TrueType", + "TypeScript", + "WebAssembly", + "WebGL", + "WebGL2", + "WebGPU", + "WebP", + "WebRTC", + "WebSocket", + "WebTransport", + "YCbCr", + "iOS", + "macOS", + "sRGB", +] msrv = "1.93.1" diff --git a/deployment-examples/docker-compose/README.md b/deployment-examples/docker-compose/README.md index 35918d91d..e3dad5f53 100644 --- a/deployment-examples/docker-compose/README.md +++ b/deployment-examples/docker-compose/README.md @@ -121,6 +121,7 @@ services: ### Single Worker Setup - [`docker-compose.yml`](./docker-compose.yml) - Docker Compose file for single worker deployment - [`local-storage-cas.json5`](./local-storage-cas.json5) - Local storage CAS configuration for single worker +- [`local-storage-cas-zstd.json5`](./local-storage-cas-zstd.json5) - Same, but the CAS selects `compression_algorithm.zstd` to keep blobs as zstd at rest for byte-for-byte `--remote_cache_compression` passthrough (see the file's header comment for the dedicated-namespace and placement rules) - [`scheduler.json5`](./scheduler.json5) - Scheduler configuration for single worker deployment - [`worker.json5`](./worker.json5) - Worker configuration for single worker deployment diff --git a/deployment-examples/docker-compose/local-storage-cas-zstd.json5 b/deployment-examples/docker-compose/local-storage-cas-zstd.json5 new file mode 100644 index 000000000..541cd933b --- /dev/null +++ b/deployment-examples/docker-compose/local-storage-cas-zstd.json5 @@ -0,0 +1,241 @@ +// Variant of `local-storage-cas.json5` that stores CAS blobs as zstd streams +// at rest via the `zstd` algorithm of `compression`, instead of raw bytes +// behind `compression` with its `lz4` algorithm. +// +// The payoff over plain `compression`: a Bazel client built with +// `--remote_cache_compression` (enabled below via `remote_cache_compression` +// on the `capabilities` service) gets served the stored zstd bytes +// byte-for-byte on a cache hit — no decompress-then-recompress round trip at +// the gRPC boundary. Identity clients (no `--remote_cache_compression`) are +// unaffected: they still get plain decompressed bytes. +// +// Read this whole comment block before adapting this file. `compression` with +// the `zstd` algorithm has deployment constraints that are not optional +// footnotes; getting them wrong +// either silently corrupts a mixed namespace or forces a cache flush later. +// Full explanation: +// https://github.com/TraceMachina/nativelink/blob/main/web/apps/docs/content/docs/reference/nativelink-config/store-overview.mdx +// +// 1. DEDICATED, EMPTY NAMESPACE ONLY. The `compression.backend` below, +// when `compression_algorithm.zstd` is selected, +// (`CAS_ZSTD_CONTENT_STORE`, and the paths it writes) must never be shared +// with a store, instance, or NativeLink version that reads or writes those +// same keys as raw bytes. The digest key is identical either way, raw CAS +// values are unstructured (a raw blob can legitimately start with the +// zstd magic number), so there is no safe way to sniff which encoding an +// entry is in. That's why this file uses brand-new `*-cas-zstd` paths +// instead of the `content_path-cas` paths from `local-storage-cas.json5` — +// reusing those would mix raw and zstd values under one namespace. +// +// 2. NO IN-PLACE MIGRATION. Selecting `compression_algorithm.zstd` for an existing +// populated CAS, or turning it back off, both require a cache flush (or +// cutting over to a new namespace as done here) — not an in-place +// conversion. The same applies to a rolling/mixed-version deploy against +// one shared namespace: unsupported, because some replicas would write +// raw and others zstd to the same keys. +// +// 3. PLACEMENT: the `compression` store with `compression_algorithm.zstd` must +// be the store an instance/service points at directly for the passthrough +// optimization above to apply. Wrappers such +// as `fast_slow`, `dedup`, `existence_cache`, `cache_metrics`, `shard`, +// `ref_store`, and `size_partitioning` are fine *inside* it (they then +// operate on the physical zstd stream). A wrapper placed *outside* +// the zstd compression store (e.g. `verify` around it) is still correct, +// but forces that wrapper to decode the stream, which disables passthrough at that +// boundary. +// +// 4. CAS-ONLY. `compression_algorithm.zstd` only supports digest-keyed CAS entries; never +// point an AC store at it (`AC_MAIN_STORE` below stays plain `filesystem`). +{ + stores: [ + { + name: "CAS_MAIN_STORE", + compression: { + compression_algorithm: { + zstd: { + // Staging directory for upload validation. MUST be on the same + // filesystem as the `filesystem` backend's `content_path` below: + // that backend commits a staged upload with rename(2), which fails + // with EXDEV across filesystems. Kept on the same + // `/root/.cache/nativelink` tree here for exactly that reason. + temp_path: "/root/.cache/nativelink/tmp_path-cas-zstd-stage", + + // Reject a compressed upload over this many wire bytes with + // RESOURCE_EXHAUSTED rather than staging it. + max_compressed_upload_size: "512MiB", + + // How many uploads may hold a staged file at once. Sizes worst-case + // temp disk usage together with max_compressed_upload_size: + // max_concurrent_staged_uploads * max_compressed_upload_size + // Recompression adds no staging file (it reuses the same + // descriptor), but does hold up to max_recompression_size in memory. + // Monitor `temp_path` against the disk bound. + max_concurrent_staged_uploads: 4, + + // How many uncompressed (identity) reads and writes to admit at + // once. Identity clients — anything without + // --remote_cache_compression — are encoded/decoded on a blocking + // thread held for the whole transfer, so this bound keeps a flood of + // slow identity clients from starving the process-wide blocking pool + // that filesystem I/O also uses. + max_concurrent_identity_ops: 256, + + // Level used to encode uploads this store compresses itself, and to + // re-encode incoming compressed uploads when + // max_recompression_size > 0. Omit for the default of 3. + // + // Valid range is 1..=19 — enforced at startup. This is *not* the + // full zstd range (up to 22): standard levels 1-19 cap the encoded + // frame's window size at windowLog <= 23 (<= 8MiB), which every + // Bazel/zstd-jni client can decode with plain libzstd defaults. + // Bazel's decoder never calls setLongMax, so its hard ceiling is + // windowLog <= 27 (128MiB) - 19 keeps a large margin under that and + // is what caps the field, not a smaller "safe zstd" limit. Long- + // distance matching and dictionaries are never enabled by this + // store. + // + // 9 is a general-purpose default here: it costs a fraction of level + // 19's CPU per byte. Raise it toward 19 for an archival cache whose + // blobs are read far more often than written. + compression_level: 9, + + // Recompression is only attempted for uploads whose *decoded* size + // is within this bound, and the smaller of the original and + // re-encoded stream is kept. 0 disables recompression entirely (an + // incoming compressed stream is always stored as-is), while still + // letting `compression_level` pick the level used for brand-new, + // non-precompressed uploads. A positive value requires + // `compression_level`; without it, startup fails rather than + // silently doing nothing. + max_recompression_size: "64MiB", + + // Concurrent recompressions admitted by this store instance. + // Recompression is best-effort: an upload that finds every slot busy + // commits its original stream rather than queueing, so a small pool + // here throttles recompression without throttling uploads. + max_concurrent_recompressions: 1, + + // Compressed uploads at or below this size are validated and + // committed straight from memory, with no staging file and no + // fsync. BatchUpdateBlobs payloads are small and numerous, so a + // per-blob disk round trip would dominate their cost. + max_inline_commit_size: "4MiB", + + // Total time one upload may spend being validated and staged, + // measured from admission to a staging slot. Unlike the ByteStream + // per-message idle timeout, continuous slow progress does NOT reset + // it, so this is the bound on a client that trickles bytes to hold a + // staging slot open. Size it against max_compressed_upload_size and + // the slowest upload bandwidth worth serving. + stage_timeout_s: 600, + + // Time budget for a staged upload's optional recompression plus the + // inner-store commit, after the client stream is done. Bounds a + // stalled backend holding a staging slot. + commit_timeout_s: 300, + }, + }, + + // MUST be a brand-new/empty namespace — see note 1 above. Everything + // under `backend` here is a complete, dedicated CAS stack; nothing + // else in this file (or any other deployment) may read or write it + // as raw bytes. + backend: { + fast_slow: { + fast: { + memory: { + eviction_policy: { + // 500mb. Absorbs hot re-reads without touching disk. + max_bytes: 500000000, + }, + }, + }, + slow: { + filesystem: { + content_path: "/root/.cache/nativelink/content_path-cas-zstd", + temp_path: "/root/.cache/nativelink/tmp_path-cas-zstd", + eviction_policy: { + // 10gb. + max_bytes: 10000000000, + }, + }, + }, + }, + }, + }, + }, + { + // Holds blob-to-chunks layouts for the SplitBlob/SpliceBlob RPCs used + // by Bazel's --experimental_remote_cache_chunking. Must not verify + // digests and must not be the same store as the CAS. + name: "CHUNK_INDEX_STORE", + filesystem: { + content_path: "/root/.cache/nativelink/content_path-chunk-index", + temp_path: "/root/.cache/nativelink/tmp_path-chunk-index", + eviction_policy: { + // 100mb. + max_bytes: 100000000, + }, + }, + }, + { + // Plain filesystem store: zstd compression is CAS-only (note 4 above), so + // the AC never goes anywhere near it. + name: "AC_MAIN_STORE", + filesystem: { + content_path: "/root/.cache/nativelink/content_path-ac", + temp_path: "/root/.cache/nativelink/tmp_path-ac", + eviction_policy: { + // 500mb. + max_bytes: 500000000, + }, + }, + }, + ], + servers: [ + { + listener: { + http: { + socket_address: "0.0.0.0:50051", + }, + }, + services: { + cas: [ + { + cas_store: "CAS_MAIN_STORE", + + // Optional: enables content-defined chunking + // (SplitBlob/SpliceBlob) for Bazel clients running with + // --experimental_remote_cache_chunking. + experimental_chunking: { + index_store: "CHUNK_INDEX_STORE", + }, + }, + ], + ac: [ + { + ac_store: "AC_MAIN_STORE", + }, + ], + capabilities: [ + { + // Advertises zstd to clients so ByteStream/CAS RPCs accept and + // serve compressed-blobs/zstd/... payloads. This is what makes + // the byte-for-byte passthrough in the zstd compression store reachable — + // without it, every client uses identity transfers and + // zstd compression still stores zstd at rest, but every read pays + // a decompress. + remote_cache_compression: true, + }, + ], + bytestream: { + cas_stores: { + "": "CAS_MAIN_STORE", + }, + }, + fetch: {}, + push: {}, + }, + }, + ], +} diff --git a/deployment-examples/docker-compose/scheduler-multi-worker.json5 b/deployment-examples/docker-compose/scheduler-multi-worker.json5 index 18a28333f..380f73b6f 100644 --- a/deployment-examples/docker-compose/scheduler-multi-worker.json5 +++ b/deployment-examples/docker-compose/scheduler-multi-worker.json5 @@ -3,7 +3,7 @@ { name: "GRPC_LOCAL_STORE", - // Note: This file is used to test GRPC store. + // Note: This file is used to test gRPC store. grpc: { instance_name: "", endpoints: [ @@ -17,7 +17,7 @@ { name: "GRPC_LOCAL_AC_STORE", - // Note: This file is used to test GRPC store. + // Note: This file is used to test gRPC store. grpc: { instance_name: "", endpoints: [ diff --git a/deployment-examples/docker-compose/scheduler.json5 b/deployment-examples/docker-compose/scheduler.json5 index 18a28333f..380f73b6f 100644 --- a/deployment-examples/docker-compose/scheduler.json5 +++ b/deployment-examples/docker-compose/scheduler.json5 @@ -3,7 +3,7 @@ { name: "GRPC_LOCAL_STORE", - // Note: This file is used to test GRPC store. + // Note: This file is used to test gRPC store. grpc: { instance_name: "", endpoints: [ @@ -17,7 +17,7 @@ { name: "GRPC_LOCAL_AC_STORE", - // Note: This file is used to test GRPC store. + // Note: This file is used to test gRPC store. grpc: { instance_name: "", endpoints: [ diff --git a/deployment-examples/docker-compose/worker-shared-cas.json5 b/deployment-examples/docker-compose/worker-shared-cas.json5 index 1198cde34..7c8aedc49 100644 --- a/deployment-examples/docker-compose/worker-shared-cas.json5 +++ b/deployment-examples/docker-compose/worker-shared-cas.json5 @@ -3,7 +3,7 @@ { name: "GRPC_LOCAL_STORE", - // Note: This file is used to test GRPC store. + // Note: This file is used to test gRPC store. grpc: { instance_name: "", endpoints: [ @@ -17,7 +17,7 @@ { name: "GRPC_LOCAL_AC_STORE", - // Note: This file is used to test GRPC store. + // Note: This file is used to test gRPC store. grpc: { instance_name: "", endpoints: [ diff --git a/deployment-examples/docker-compose/worker.json5 b/deployment-examples/docker-compose/worker.json5 index 7e55b0f17..ae2afb881 100644 --- a/deployment-examples/docker-compose/worker.json5 +++ b/deployment-examples/docker-compose/worker.json5 @@ -3,7 +3,7 @@ { name: "GRPC_LOCAL_STORE", - // Note: This file is used to test GRPC store. + // Note: This file is used to test gRPC store. grpc: { instance_name: "", endpoints: [ @@ -17,7 +17,7 @@ { name: "GRPC_LOCAL_AC_STORE", - // Note: This file is used to test GRPC store. + // Note: This file is used to test gRPC store. grpc: { instance_name: "", endpoints: [ diff --git a/flake.nix b/flake.nix index dee3e0b77..0b71c013f 100644 --- a/flake.nix +++ b/flake.nix @@ -515,6 +515,30 @@ unset TMPDIR TMP exec ${pkgs.bazelisk}/bin/bazelisk "$@" ''; + bazel-retry = pkgs.writeShellScriptBin "bazel-retry" '' + set -o pipefail + BAZEL_LOG=$(mktemp -t) + unset TMPDIR TMP + # Bazel's downloader retry only covers truncated downloads, not HTTP 5xx/403 failures, + # so a flaky fetch aborts the build with no retry. We retry the download in this case. + delay=5 + for attempt in 1 2 3; do + if exec ${pkgs.bazelisk}/bin/bazelisk "$@" | tee ''${BAZEL_LOG}; then + exit 0 + fi + grep -E '^ERROR:' ''${BAZEL_LOG} \ + | grep -Eq 'GET returned (403|429|5[0-9][0-9])|Bad Gateway|Connection (reset|timed out)|read timed out|Could not resolve host' || { + echo "Bazel failed (non-transient); not retrying" + exit 1 + } + [ "$attempt" -lt 3 ] || break + echo "Transient fetch error (attempt ''${attempt}); retrying in ''${delay}s..." + sleep "$delay" + delay=$((delay * 2)) + done + echo "Bazel still failing after 3 attempts." + exit 1 + ''; in [ # Development tooling @@ -527,6 +551,7 @@ # Rust bazel + bazel-retry pkgs.lre.stable-rust pkgs.lre.lre-rs.lre-rs-configs-gen pkgs.rust-analyzer diff --git a/integration_tests/generate_redis_keys.py b/integration_tests/generate_redis_keys.py new file mode 100644 index 000000000..4b954f45c --- /dev/null +++ b/integration_tests/generate_redis_keys.py @@ -0,0 +1,6 @@ +from sys import stderr + + +for i in range(1000000): + print('set name'+str(i),'helloworld') + print(i, file=stderr) diff --git a/kubernetes/components/worker/worker.json5 b/kubernetes/components/worker/worker.json5 index d68c57d55..01f729bb3 100644 --- a/kubernetes/components/worker/worker.json5 +++ b/kubernetes/components/worker/worker.json5 @@ -3,7 +3,7 @@ { name: "GRPC_LOCAL_STORE", - // Note: This file is used to test GRPC store. + // Note: This file is used to test gRPC store. grpc: { instance_name: "", endpoints: [ @@ -17,7 +17,7 @@ { name: "GRPC_LOCAL_AC_STORE", - // Note: This file is used to test GRPC store. + // Note: This file is used to test gRPC store. grpc: { instance_name: "", endpoints: [ diff --git a/local-remote-execution/MODULE.bazel b/local-remote-execution/MODULE.bazel index 8c95cf2ec..a4ecad5a8 100644 --- a/local-remote-execution/MODULE.bazel +++ b/local-remote-execution/MODULE.bazel @@ -10,11 +10,11 @@ module( bazel_dep(name = "platforms", version = "1.1.0") # Use the starlark implementation of C++ rules instead of the builtin ones. -bazel_dep(name = "rules_cc", version = "0.2.18") +bazel_dep(name = "rules_cc", version = "0.2.19") # Use the starlark implementation of Java rules instead of the builtin ones. bazel_dep(name = "rules_java", version = "8.11.0") -bazel_dep(name = "rules_rs", version = "0.0.76") +bazel_dep(name = "rules_rs", version = "0.0.96") bazel_dep(name = "rules_rust", version = "0.68.1") bazel_dep(name = "bazel_skylib", version = "1.9.0") # rules_rust is pinned to the hermeticbuild fork in the parent MODULE.bazel via diff --git a/nativelink-config/Cargo.toml b/nativelink-config/Cargo.toml index bb46a363a..736bf2abc 100644 --- a/nativelink-config/Cargo.toml +++ b/nativelink-config/Cargo.toml @@ -4,7 +4,7 @@ lints.workspace = true [package] edition = "2024" name = "nativelink-config" -version = "1.6.1" +version = "1.6.3" [dependencies] nativelink-error = { path = "../nativelink-error" } diff --git a/nativelink-config/examples/redis.json5 b/nativelink-config/examples/redis.json5 index 082ea6cd6..476ac42a2 100644 --- a/nativelink-config/examples/redis.json5 +++ b/nativelink-config/examples/redis.json5 @@ -6,7 +6,7 @@ addresses: [ "redis://127.0.0.1:6379/", ], - mode: "cluster", + mode: "standard", }, }, { @@ -15,7 +15,7 @@ addresses: [ "redis://127.0.0.1:6379/", ], - mode: "cluster", + mode: "standard", }, }, { diff --git a/nativelink-config/examples/stores-config.json5 b/nativelink-config/examples/stores-config.json5 index 0748ce406..fc6922369 100644 --- a/nativelink-config/examples/stores-config.json5 +++ b/nativelink-config/examples/stores-config.json5 @@ -172,7 +172,7 @@ "content_path": "/tmp/nativelink/data/content_path-cas", "temp_path": "/tmp/nativelink/data/tmp_path-cas", "eviction_policy": { - "max_bytes": "2gb", + "max_bytes": "2gb" } } } @@ -180,6 +180,30 @@ }, { name: "12", + "compression": { + "compression_algorithm": { + "zstd": { + "temp_path": "/var/tmp/nativelink-zstd", + "max_compressed_upload_size": "512MiB", + "max_concurrent_staged_uploads": 4, + "max_concurrent_identity_ops": 256, + "compression_level": 9, + "max_recompression_size": "64MiB", + "max_concurrent_recompressions": 1, + "max_inline_commit_size": "4MiB", + "stage_timeout_s": 600, + "commit_timeout_s": 300 + } + }, + "backend": { + "memory": { + "eviction_policy": { "max_bytes": "10GiB" } + } + } + } + }, + { + name: "13", "dedup": { "index_store": { "memory": { @@ -218,7 +242,7 @@ } }, { - name: "13", + name: "14", "existence_cache": { "backend": { "memory": { @@ -234,7 +258,7 @@ } }, { - name: "14", + name: "15", "fast_slow": { "fast": { "filesystem": { @@ -257,7 +281,7 @@ } }, { - name: "15", + name: "16", "shard": { "stores": [ { @@ -273,7 +297,7 @@ } }, { - name: "16", + name: "17", "filesystem": { "content_path": "/tmp/nativelink/data-worker-test/content_path-cas", "temp_path": "/tmp/nativelink/data-worker-test/tmp_path-cas", @@ -283,13 +307,13 @@ } }, { - name: "17", + name: "18", "ref_store": { "name": "FS_CONTENT_STORE" } }, { - name: "18", + name: "19", "size_partitioning": { "size": "128mib", "lower_store": { @@ -306,7 +330,7 @@ } }, { - name: "19", + name: "20", "grpc": { "instance_name": "main", "endpoints": [ @@ -327,7 +351,7 @@ } }, { - name: "20", + name: "21", "redis_store": { "addresses": [ "redis://127.0.0.1:6379/", @@ -336,11 +360,11 @@ } }, { - name: "21", + name: "22", "noop": {} }, { - name: "22", + name: "23", "experimental_mongo": { "connection_string": "mongodb://localhost:27017", "database": "nativelink", diff --git a/nativelink-config/src/backcompat.rs b/nativelink-config/src/backcompat.rs index 00b9e9b3e..18c9ea8f5 100644 --- a/nativelink-config/src/backcompat.rs +++ b/nativelink-config/src/backcompat.rs @@ -102,6 +102,8 @@ where max_bytes_per_stream: old_config.max_bytes_per_stream, persist_stream_on_disconnect_timeout_s: old_config .persist_stream_on_disconnect_timeout_s, + // Not expressible in the deprecated config; take the default. + compressed_upload_idle_timeout_s: 0, }, }) .collect(); diff --git a/nativelink-config/src/cas_server.rs b/nativelink-config/src/cas_server.rs index de7213f1b..2e9d4f18e 100644 --- a/nativelink-config/src/cas_server.rs +++ b/nativelink-config/src/cas_server.rs @@ -349,10 +349,25 @@ pub struct ByteStreamConfig { alias = "persist_stream_on_disconnect_timeout" )] pub persist_stream_on_disconnect_timeout_s: usize, + + /// How long, in seconds, to wait for the *next* `WriteRequest` of a + /// compressed (`compressed-blobs/...`) upload before failing it with + /// `DEADLINE_EXCEEDED`. A client that keeps making progress resets the + /// deadline on every message, so this never rejects a large upload — it + /// bounds a client that opens a compressed write, takes a store slot, then + /// stalls or never sends `finish_write`. + /// + /// Default: 60 seconds + #[serde( + default, + deserialize_with = "convert_duration_with_shellexpand", + skip_serializing_if = "is_default" + )] + pub compressed_upload_idle_timeout_s: usize, } // Older bytestream config. All fields are as per the newer docs, but this requires -// the hashed cas_stores v.s. the WithInstanceName approach. This should _not_ be updated +// the hashed `cas_stores` v.s. the WithInstanceName approach. This should _not_ be updated // with newer fields, and eventually dropped #[derive(Deserialize, Serialize, Debug, Clone)] #[serde(deny_unknown_fields)] @@ -488,7 +503,7 @@ pub struct ServicesConfig { /// Capabilities service is required in order to use most of the /// bazel protocol. This service is used to provide the supported - /// features and versions of this bazel GRPC service. + /// features and versions of this bazel gRPC service. #[serde( default, deserialize_with = "super::backcompat::opt_vec_with_instance_name" @@ -878,7 +893,7 @@ pub struct UploadActionResultConfig { pub struct LocalWorkerConfig { /// Name of the worker. This is give a more friendly name to a worker for logging /// and metric publishing. This is also the prefix of the worker id - /// (ie: "{name}{uuidv6}"). + /// (i.e. "{name}{uuidv6}"). /// Default: {Index position in the workers list} #[serde(default, deserialize_with = "convert_string_with_shellexpand")] pub name: String, @@ -994,7 +1009,7 @@ pub struct LocalWorkerConfig { pub platform_properties: HashMap, /// An optional mapping of environment names to set for the execution - /// as well as those specified in the action itself. If set, will set each + /// as well as those specified in the action itself. If set, will set each /// key as an environment variable before executing the job with the value /// of the environment variable being the value of the property of the /// action being executed of that name or the fixed value. @@ -1006,9 +1021,9 @@ pub struct LocalWorkerConfig { /// Default: None (directory cache disabled) pub directory_cache: Option, - /// Whether to use namespaces to isolate the execution. This is only available - /// on Linux. It is highly recommended as it avoids a number of issues with - /// zombie processes and also provides additional hermeticity. If explicitly set + /// Whether to use namespaces to isolate the execution. This is only available + /// on Linux. It is highly recommended as it avoids a number of issues with + /// zombie processes and also provides additional hermeticity. If explicitly set /// to true and it is not supported the worker will exit with an error. /// /// Note: this will fail for non-privileged Dockerised workers, as workers in @@ -1018,9 +1033,9 @@ pub struct LocalWorkerConfig { /// Default: False. pub use_namespaces: Option, - /// Whether to use a mount namespace to isolate the worker root. This is only - /// available on Linux and when `use_namespaces` is true. It is highly recommended - /// provides additional hermeticity. If explicitly set to true and it is not + /// Whether to use a mount namespace to isolate the worker root. This is only + /// available on Linux and when `use_namespaces` is true. It is highly recommended + /// provides additional hermeticity. If explicitly set to true and it is not /// supported or `use_namespaces` is not set to true the worker will exit with an /// error. /// Default: False. @@ -1090,6 +1105,25 @@ pub struct DirectoryCacheConfig { /// Default: 64 #[serde(default = "default_directory_cache_max_concurrent_fetches")] pub max_concurrent_fetches: usize, + /// On a directory-cache miss, prefetch every `Directory` proto of the + /// tree with one logical `GetTree` traversal instead of fetching protos + /// level by level (which costs one round trip per tree DEPTH). The + /// [REAPI request contract] permits servers to impose their own limit even + /// when no page size is specified, and the [REAPI response contract] + /// requires clients to continue with the returned page token. A paginated + /// traversal may therefore use multiple RPCs. Only takes effect when the + /// slow tier is a `grpc` store; + /// any prefetch failure falls back to the per-level path. Worthwhile when + /// worker-to-CAS latency is non-trivial and trees are deep; measured 5-34x + /// on the proto phase at 5-25ms RTT. Note the serving CAS pays the tree walk + /// against its own backend, so its directory protos should be served + /// from a fast tier. + /// + /// [REAPI request contract]: https://github.com/bazelbuild/remote-apis/blob/becdd8f9ff811df88a22d3eadd6341753d51d167/build/bazel/remote/execution/v2/remote_execution.proto#L1932-L1942 + /// [REAPI response contract]: https://github.com/bazelbuild/remote-apis/blob/becdd8f9ff811df88a22d3eadd6341753d51d167/build/bazel/remote/execution/v2/remote_execution.proto#L1961-L1965 + /// Default: false + #[serde(default)] + pub experimental_get_tree_prefetch: bool, } const fn default_directory_cache_max_entries() -> usize { diff --git a/nativelink-config/src/schedulers.rs b/nativelink-config/src/schedulers.rs index 3d32c9f33..3b840e545 100644 --- a/nativelink-config/src/schedulers.rs +++ b/nativelink-config/src/schedulers.rs @@ -83,6 +83,11 @@ const fn default_worker_match_logging_interval_s() -> i64 { 10 } +// defaults to every 5s +const fn default_fallback_match_interval_s() -> i64 { + 5 +} + #[derive(Deserialize, Serialize, Debug, Default)] #[serde(deny_unknown_fields)] #[cfg_attr(feature = "dev-schema", derive(JsonSchema))] @@ -100,7 +105,7 @@ pub struct SimpleSpec { /// { "cpu_count": "8", "cpu_arch": "arm" } /// ``` /// Will result in the scheduler filtering out any workers that do not have - /// `"cpu_arch" = "arm"` and filter out any workers that have less than 8 cpu + /// `"cpu_arch" = "arm"` and filter out any workers that have less than 8 CPU /// cores available. /// /// The property names here must match the property keys provided by the @@ -108,7 +113,7 @@ pub struct SimpleSpec { /// publish their capabilities to the scheduler when they join the worker /// pool. If the worker fails to notify the scheduler of its (for example) /// `"cpu_arch"`, the scheduler will never send any jobs to it, if all jobs - /// have the `"cpu_arch"` label. There is no special treatment of any platform + /// have the `"cpu_arch"` label. We have no special treatment of any platform /// property labels other and entirely driven by worker configs and this /// config. pub supported_platform_properties: Option>, @@ -167,6 +172,20 @@ pub struct SimpleSpec { deserialize_with = "convert_duration_with_shellexpand_and_negative" )] pub worker_match_logging_interval_s: i64, + + /// Every N seconds, run a worker matching pass even if no task or worker + /// change notification arrived. This is a safety net for missed + /// notifications and for scheduler backends with eventually consistent + /// searches (for example Redis), where an operation that was re-queued + /// may not be visible to the search triggered by its own notification. + /// Without this, such an operation can stay queued until an unrelated + /// event triggers another matching pass. + /// Defaults to 5s. Zero or any negative value disables it. + #[serde( + default = "default_fallback_match_interval_s", + deserialize_with = "convert_duration_with_shellexpand_and_negative" + )] + pub fallback_match_interval_s: i64, } #[derive(Deserialize, Serialize, Debug)] @@ -188,9 +207,9 @@ pub struct ExperimentalRedisSchedulerBackend { pub redis_store: StoreRefName, } -/// A scheduler that simply forwards requests to an upstream scheduler. This +/// A scheduler that forwards requests to an upstream scheduler. This /// is useful to use when doing some kind of local action cache or CAS away from -/// the main cluster of workers. In general, it's more efficient to point the +/// the main cluster of workers. In general, it's more efficient to point the /// build at the main scheduler directly though. #[derive(Deserialize, Serialize, Debug)] #[serde(deny_unknown_fields)] @@ -203,8 +222,8 @@ pub struct GrpcSpec { #[serde(default)] pub retry: Retry, - /// Limit the number of simultaneous upstream requests to this many. A - /// value of zero is treated as unlimited. If the limit is reached the + /// Limit the number of simultaneous upstream requests to this many. A + /// value of zero is treated as unlimited. If the limit is reached the /// request is queued. /// Default: unlimited #[serde(default, deserialize_with = "convert_numeric_with_shellexpand")] @@ -246,7 +265,7 @@ pub struct PlatformPropertyAddition { pub struct PlatformPropertyReplacement { /// The name of the property to replace. pub name: String, - /// The the value to match against, if unset then any instance matches. + /// The value to match against, if unset then any instance matches. #[serde(default)] pub value: Option, /// The new name of the property. @@ -273,9 +292,9 @@ pub enum PropertyModification { #[cfg_attr(feature = "dev-schema", derive(JsonSchema))] pub struct PropertyModifierSpec { /// A list of modifications to perform to incoming actions for the nested - /// scheduler. These are performed in order and blindly, so removing a + /// scheduler. These are performed in order and blindly, so removing a /// property that doesn't exist is fine and overwriting an existing property - /// is also fine. If adding properties that do not exist in the nested + /// is also fine. If adding properties that do not exist in the nested /// scheduler is not supported and will likely cause unexpected behaviour. pub modifications: Vec, diff --git a/nativelink-config/src/stores.rs b/nativelink-config/src/stores.rs index cfa69715b..26984d32e 100644 --- a/nativelink-config/src/stores.rs +++ b/nativelink-config/src/stores.rs @@ -153,8 +153,8 @@ pub enum StoreSpec { /// } /// ``` /// - /// 4. **`NetApp` ONTAP S3** - /// `NetApp` ONTAP S3 store will use ONTAP's S3-compatible storage as a backend + /// 4. **NetApp ONTAP S3:** + /// NetApp ONTAP S3 store will use ONTAP's S3-compatible storage as a backend /// to store files. This store is specifically configured for ONTAP's S3 requirements /// including custom TLS configuration, credentials management, and proper vserver /// configuration. @@ -315,7 +315,7 @@ pub enum StoreSpec { /// is a concern it is often faster and more efficient to use this /// store before those stores. /// - /// **Example JSON Config:** + /// **LZ4 example:** /// ```json /// "compression": { /// "compression_algorithm": { @@ -326,13 +326,50 @@ pub enum StoreSpec { /// "content_path": "/tmp/nativelink/data/content_path-cas", /// "temp_path": "/tmp/nativelink/data/tmp_path-cas", /// "eviction_policy": { - /// "max_bytes": "2gb", + /// "max_bytes": "2gb" /// } /// } /// } /// } /// ``` /// + /// **Zstd example:** + /// ```json + /// "compression": { + /// "compression_algorithm": { + /// "zstd": { + /// "temp_path": "/var/tmp/nativelink-zstd", + /// "max_compressed_upload_size": "512MiB", + /// "max_concurrent_staged_uploads": 4, + /// "max_concurrent_identity_ops": 256, + /// "compression_level": 9, + /// "max_recompression_size": "64MiB", + /// "max_concurrent_recompressions": 1, + /// "max_inline_commit_size": "4MiB", + /// "stage_timeout_s": 600, + /// "commit_timeout_s": 300 + /// } + /// }, + /// "backend": { + /// "memory": { + /// "eviction_policy": { "max_bytes": "10GiB" } + /// } + /// } + /// } + /// ``` + /// + /// The `zstd` algorithm keeps CAS blobs as zstd streams at rest and serves them + /// byte-for-byte to `--remote_cache_compression` clients. Its `backend` MUST be + /// a new or empty dedicated namespace, never shared with processes that read or + /// write the same keys as raw bytes. Rollout and rollback require a cache flush + /// or new namespace — there is no in-place migration. + /// + /// For byte-for-byte passthrough, this compression store must be the store the + /// instance points at directly. `fast_slow`, `dedup`, `existence_cache`, + /// `cache_metrics`, `shard`, `ref`, and `size_partitioning` may appear **inside** + /// its `backend`. Any wrapper **outside** it is correct but disables passthrough + /// at that boundary. + /// Compression(Box), /// A dedup store will take the inputs and run a rolling hash @@ -438,7 +475,7 @@ pub enum StoreSpec { /// WARNING: If you need data to always exist in the `slow` store /// for something like remote execution, be careful because this /// store will never check to see if the objects exist in the - /// `slow` store if it exists in the `fast` store (ie: it assumes + /// `slow` store if it exists in the `fast` store (i.e. it assumes /// that if an object exists in the `fast` store it will exist in /// the `slow` store). /// @@ -531,7 +568,7 @@ pub enum StoreSpec { /// used if the size field is the real size of the content, in other /// words, don't use on AC (Action Cache) stores. Any store where you can /// safely use `VerifySpec.verify_size = true`, this store should be safe - /// to use (ie: CAS stores). + /// to use (i.e. CAS stores). /// /// **Example JSON Config:** /// ```json @@ -553,7 +590,7 @@ pub enum StoreSpec { /// SizePartitioning(Box), - /// This store will pass-through calls to another GRPC store. This store + /// This store will pass-through calls to another gRPC store. This store /// is not designed to be used as a sub-store of another store, but it /// does satisfy the interface and will likely work. /// @@ -593,6 +630,12 @@ pub enum StoreSpec { /// Ideal for accepting small object sizes as most redis store /// services have a max file upload of between 256Mb-512Mb. /// + /// If you are using Redis together with any stores above it + /// e.g. existence cache, you will need to configure `notify-keyspace-events` + /// to `KA` as per + /// in order for us to get eviction events. Failing to do so will get you + /// log messages complaining about it, as well as errors like + /// /// **Example JSON Config:** /// ```json /// "redis_store": { @@ -707,7 +750,7 @@ pub struct RefSpec { pub struct FilesystemSpec { /// Path on the system where to store the actual content. This is where /// the bulk of the data will be placed. - /// On service bootup this folder will be scanned and all files will be + /// On service boot this folder will be scanned and all files will be /// added to the cache. In the event one of the files doesn't match the /// criteria, the file will be deleted. #[serde(deserialize_with = "convert_string_with_shellexpand")] @@ -716,7 +759,7 @@ pub struct FilesystemSpec { /// A temporary location of where files that are being uploaded or /// deleted will be placed while the content cannot be guaranteed to be /// accurate. This location must be on the same block device as - /// `content_path` so atomic moves can happen (ie: move without copy). + /// `content_path` so atomic moves can happen (i.e. move without copy). /// All files in this folder will be deleted on every startup. #[serde(deserialize_with = "convert_string_with_shellexpand")] pub temp_path: String, @@ -748,6 +791,16 @@ pub struct FilesystemSpec { /// Default: unlimited #[serde(default, deserialize_with = "convert_numeric_with_shellexpand")] pub max_concurrent_writes: usize, + + /// When true, advise the kernel to drop the page cache for each blob after + /// it is written or read (`posix_fadvise` with `POSIX_FADV_DONTNEED`). On + /// real filesystems this takes a globally serialized, all-CPU kernel path + /// that stalls on many-core hosts, and it evicts the page-cache tier that + /// fast-disk deployments rely on. Leave off unless you specifically want to + /// keep this store's I/O out of the page cache. + /// Default: false + #[serde(default)] + pub evict_page_cache: bool, } // NetApp ONTAP S3 Spec @@ -883,7 +936,7 @@ pub struct FastSlowSpec { /// out to the `slow` store. pub fast: StoreSpec, - /// How to handle the fast store. This can be useful to set to Get for + /// How to handle the fast store. This can be useful to set to Get for /// worker nodes such that results are persisted to the slow store only. #[serde(default)] pub fast_direction: StoreDirection, @@ -892,7 +945,7 @@ pub struct FastSlowSpec { /// get it from this store. pub slow: StoreSpec, - /// How to handle the slow store. This can be useful if creating a diode + /// How to handle the slow store. This can be useful if creating a diode /// and you wish to have an upstream read only store. #[serde(default)] pub slow_direction: StoreDirection, @@ -963,7 +1016,7 @@ pub struct DedupSpec { /// Due to implementation detail, we want to prefer to download /// the first chunks of the file so we can stream the content /// out and free up some of our buffers. This configuration - /// will be used to to restrict the number of concurrent chunk + /// will be used to restrict the number of concurrent chunk /// downloads at a time per `get()` request. /// /// This setting will also affect how much memory might be used @@ -1055,7 +1108,7 @@ pub struct Lz4Config { pub max_decode_block_size: u32, } -#[derive(Serialize, Deserialize, Debug, PartialEq, Eq, Clone, Copy)] +#[derive(Serialize, Deserialize, Debug, PartialEq, Eq, Clone)] #[serde(rename_all = "snake_case")] #[cfg_attr(feature = "dev-schema", derive(JsonSchema))] pub enum CompressionAlgorithm { @@ -1067,6 +1120,11 @@ pub enum CompressionAlgorithm { /// /// see: Lz4(Lz4Config), + + /// Zstd compression keeps blobs as standard zstd streams at rest. When this + /// compression store is directly configured for an instance, zstd wire-compression + /// clients can receive the stored stream byte-for-byte. + Zstd(ZstdConfig), } #[derive(Serialize, Deserialize, Debug, Clone)] @@ -1084,6 +1142,88 @@ pub struct CompressionSpec { pub compression_algorithm: CompressionAlgorithm, } +#[derive(Serialize, Deserialize, Debug, Default, PartialEq, Eq, Clone)] +#[serde(deny_unknown_fields)] +#[cfg_attr(feature = "dev-schema", derive(JsonSchema))] +pub struct ZstdConfig { + /// Operator-controlled staging directory for validation. Put it on the same + /// filesystem as the `content_path` of a `filesystem` backend: that backend + /// commits by `rename(2)`, which fails with `EXDEV` across filesystems. + #[serde(default, deserialize_with = "convert_string_with_shellexpand")] + pub temp_path: String, + + /// Max compressed wire bytes accepted by an upload. Exceeding it returns + /// `RESOURCE_EXHAUSTED`. Accepts "512MiB"-style strings. + #[serde(default, deserialize_with = "convert_data_size_with_shellexpand")] + pub max_compressed_upload_size: u64, + + /// Max uploads holding staging files at once. `0` means "use default 4" + /// at construction time. + /// Default: 0 + #[serde(default, deserialize_with = "convert_numeric_with_shellexpand")] + pub max_concurrent_staged_uploads: usize, + + /// Max concurrent uncompressed (identity) reads and writes this store will + /// admit. Each one occupies a blocking thread for the whole transfer, so + /// this bound keeps a flood of slow identity clients from starving the + /// process-wide blocking pool that filesystem I/O also uses. `0` means "use + /// default 256" at construction time. + /// Default: 0 + #[serde(default, deserialize_with = "convert_numeric_with_shellexpand")] + pub max_concurrent_identity_ops: usize, + + /// Level used to encode uploads this store compresses itself (identity + /// uploads) and, when `max_recompression_size > 0`, to re-encode incoming + /// compressed uploads. Omitted uses level 3. Validated `1..=19` at startup. + pub compression_level: Option, + + /// Max uncompressed blob size eligible for optional recompression of an + /// already-compressed upload. `0` disables recompression. Requires + /// `compression_level` to be set; a positive value without it is rejected + /// at startup rather than silently doing nothing. + #[serde(default, deserialize_with = "convert_data_size_with_shellexpand")] + pub max_recompression_size: u64, + + /// Concurrent recompressions admitted by this store. Recompression is + /// best-effort: an upload that finds every slot busy commits the client's + /// original stream instead of queueing behind them, so this never blocks a + /// staging slot. `0` means "use default 1" at construction time. + /// Default: 0 + #[serde(default, deserialize_with = "convert_numeric_with_shellexpand")] + pub max_concurrent_recompressions: usize, + + /// Maximum time, in seconds, one upload may spend being validated and + /// staged, measured from the moment it is admitted to a staging slot. This + /// is the bound on a client that trickles bytes to hold a slot open: unlike + /// a per-message idle timeout, continuous slow progress does not reset it. + /// On expiry it fails with `DEADLINE_EXCEEDED`, removes the staged file, and + /// releases its slot. Size it against `max_compressed_upload_size` and the + /// slowest upload bandwidth worth serving. + /// `0` means "use default 600" at construction time. + /// Default: 0 + #[serde(default, deserialize_with = "convert_numeric_with_shellexpand")] + pub stage_timeout_s: u64, + + /// Maximum time, in seconds, allowed for the optional recompression of a + /// staged upload plus its inner-store commit. The timer starts after the + /// client stream has finished validation and staging, so it does not reject + /// a steadily progressing large upload. On expiry it fails with + /// `DEADLINE_EXCEEDED`, removes the staged file, and releases its slot. + /// `0` means "use default 300" at construction time. + /// Default: 0 + #[serde(default, deserialize_with = "convert_numeric_with_shellexpand")] + pub commit_timeout_s: u64, + + /// Compressed uploads at or below this many bytes are validated and + /// committed straight from memory, with no staging file and no `fsync`. + /// `BatchUpdateBlobs` payloads are small and numerous, so writing each one + /// to disk would dominate their cost. Larger uploads always stage to + /// `temp_path`. `0` means "use default 4MiB" at construction time. + /// Default: 0 + #[serde(default, deserialize_with = "convert_data_size_with_shellexpand")] + pub max_inline_commit_size: u64, +} + /// Eviction policy always works on LRU (Least Recently Used). Any time an entry /// is touched it updates the timestamp. Inserts and updates will execute the /// eviction policy removing any expired entries and/or the oldest entries @@ -1098,7 +1238,7 @@ pub struct EvictionPolicy { pub max_bytes: usize, /// When eviction starts based on hitting `max_bytes`, continue until - /// `max_bytes - evict_bytes` is met to create a low watermark. This stops + /// `max_bytes - evict_bytes` is met to create a low watermark. This stops /// operations from thrashing when the store is close to the limit. /// Default: 0 #[serde(default, deserialize_with = "convert_data_size_with_shellexpand")] @@ -1274,10 +1414,10 @@ pub struct CommonObjectSpec { #[serde(default, deserialize_with = "convert_boolean_with_shellexpand")] pub insecure_allow_http: bool, - /// Disable http/2 connections and only use http/1.1. Default client - /// configuration will have http/1.1 and http/2 enabled for connection - /// schemes. Http/2 should be disabled if environments have poor support - /// or performance related to http/2. Safe to keep default unless + /// Disable HTTP/2 connections and only use HTTP/1.1. Default client + /// configuration will have HTTP/1.1 and HTTP/2 enabled for connection + /// schemes. HTTP/2 should be disabled if environments have poor support + /// or performance related to HTTP/2. Safe to keep default unless /// underlying network environment, S3, or GCS API servers specify otherwise. /// /// Default: false @@ -1327,7 +1467,7 @@ pub struct ClientTlsConfig { #[serde(deny_unknown_fields)] #[cfg_attr(feature = "dev-schema", derive(JsonSchema))] pub struct GrpcEndpoint { - /// The endpoint address (i.e. grpc(s)://example.com:443). + /// The endpoint address (i.e. `grpc(s)://example.com:443`). #[serde(deserialize_with = "convert_string_with_shellexpand")] pub address: String, /// The TLS configuration to use to connect to the endpoint (if grpcs). @@ -1367,7 +1507,7 @@ pub struct GrpcEndpoint { #[serde(deny_unknown_fields)] #[cfg_attr(feature = "dev-schema", derive(JsonSchema))] pub struct GrpcSpec { - /// Instance name for GRPC calls. Proxy calls will have the `instance_name` changed to this. + /// Instance name for gRPC calls. Proxy calls will have the `instance_name` changed to this. #[serde(default, deserialize_with = "convert_string_with_shellexpand")] pub instance_name: String, @@ -1381,19 +1521,20 @@ pub struct GrpcSpec { #[serde(default)] pub retry: Retry, - /// Limit the number of simultaneous upstream requests to this many. A - /// value of zero is treated as unlimited. If the limit is reached the + /// Limit the number of simultaneous upstream requests to this many. A + /// value of zero is treated as unlimited. If the limit is reached the /// request is queued. #[serde(default, deserialize_with = "convert_numeric_with_shellexpand")] pub max_concurrent_requests: usize, /// The number of connections to make to each specified endpoint to balance - /// the load over multiple TCP connections. Default 1. + /// the load over multiple TCP connections. + /// Default: 1. #[serde(default, deserialize_with = "convert_numeric_with_shellexpand")] pub connections_per_endpoint: usize, /// Maximum time (seconds) allowed for a single RPC request (e.g. a - /// ByteStream.Write call) before it is cancelled. + /// `ByteStream.Write` call) before it is cancelled. /// /// A value of 0 (the default) disables the per-RPC timeout. Dead /// connections are still detected by the HTTP/2 and TCP keepalive @@ -1439,6 +1580,114 @@ pub struct GrpcSpec { /// context (`traceparent` / `tracestate`) into every outgoing request. #[serde(default)] pub forward_headers: Vec, + + /// Optional and experimental: coalesce small-blob reads into + /// `BatchReadBlobs` RPCs instead of issuing one `ByteStream` `Read` + /// stream per blob. Each `ByteStream` read carries a fixed per-RPC cost, + /// so batching many small reads into a single `BatchReadBlobs` request + /// can dramatically reduce read latency for small blobs. + /// + /// Only full reads (offset 0, whole blob) of blobs at or below + /// `max_blob_size_bytes` are batched; everything else continues to use + /// the `ByteStream` `Read` path. + /// + /// Incompatible with `forward_headers`: batched reads share one upstream + /// RPC across many client requests, so per-client forwarded headers + /// (e.g. credentials) cannot be attached. Configuring both is rejected + /// at startup. + /// + /// Default: unset (disabled). When unset there is zero behavior change. + #[serde(default)] + pub experimental_read_batching: Option, + + /// Compress this store's own blob transfers on the wire with REAPI + /// `compressed-blobs/zstd`. Uploads and full-blob downloads of blobs at + /// or above 64 KiB are zstd-compressed; smaller blobs and ranged reads + /// keep the identity path. + /// + /// The upstream instance must have + /// `capabilities.remote_cache_compression` enabled, otherwise compressed + /// requests fail with `InvalidArgument`. This setting is what makes + /// NativeLink-to-NativeLink hops (for example worker to CAS) benefit + /// from wire compression; it is independent of what external clients + /// such as Bazel negotiate for themselves. + /// + /// Compressed uploads do not resume mid-stream (mirroring the REAPI + /// server contract): a transport failure part-way through a compressed + /// upload surfaces immediately to the caller instead of retrying, and + /// outer callers retry the whole upload. + /// + /// When combined with `experimental_chunked_uploads`, chunked uploads + /// take precedence for blobs at or above the chunking threshold. + /// + /// Default: false (disabled). + #[serde(default, deserialize_with = "convert_boolean_with_shellexpand")] + pub experimental_remote_cache_compression: bool, +} + +/// Configuration for experimental small-blob read coalescing in a gRPC +/// store. See [`GrpcSpec::experimental_read_batching`]. +#[derive(Serialize, Deserialize, Debug, Clone, Copy)] +#[serde(deny_unknown_fields)] +#[cfg_attr(feature = "dev-schema", derive(JsonSchema))] +pub struct GrpcReadBatchingConfig { + /// Only blobs at or below this size (in bytes) are eligible for + /// batching. Larger blobs always use the `ByteStream` `Read` path. + /// + /// Default: 131072 (128 KiB). + #[serde( + default = "default_read_batching_max_blob_size_bytes", + deserialize_with = "convert_data_size_with_shellexpand" + )] + pub max_blob_size_bytes: u64, + + /// Maximum total payload bytes packed into a single `BatchReadBlobs` + /// request. This should leave headroom under the 4 MiB default gRPC + /// message limit for protobuf framing overhead. + /// + /// Default: 3145728 (3 MiB). + #[serde( + default = "default_read_batching_max_batch_bytes", + deserialize_with = "convert_data_size_with_shellexpand" + )] + pub max_batch_bytes: u64, + + /// Maximum number of concurrent `BatchReadBlobs` RPCs dispatched by the + /// coalescer. Must be greater than zero. + /// + /// Default: 4. + #[serde( + default = "default_read_batching_dispatch_slots", + deserialize_with = "convert_numeric_with_shellexpand" + )] + pub dispatch_slots: usize, + + /// Bound on the number of payload bytes waiting in the coalescer queue. + /// When exceeded, new read requests bypass batching and fall back to the + /// regular `ByteStream` `Read` path instead of blocking. + /// + /// Default: 33554432 (32 MiB). + #[serde( + default = "default_read_batching_max_queued_bytes", + deserialize_with = "convert_data_size_with_shellexpand" + )] + pub max_queued_bytes: u64, +} + +const fn default_read_batching_max_blob_size_bytes() -> u64 { + 128 * 1024 // 128 KiB. +} + +const fn default_read_batching_max_batch_bytes() -> u64 { + 3 * 1024 * 1024 // 3 MiB. +} + +const fn default_read_batching_dispatch_slots() -> usize { + 4 +} + +const fn default_read_batching_max_queued_bytes() -> u64 { + 32 * 1024 * 1024 // 32 MiB. } /// The possible error codes that might occur on an upstream request. @@ -1558,7 +1807,7 @@ pub struct RedisSpec { #[serde(default, deserialize_with = "convert_numeric_with_shellexpand")] pub read_chunk_size: usize, - /// The number of connections to keep open to the redis server(s). + /// The number of connections to keep open to the redis servers. /// /// Default: 3 #[serde(default, deserialize_with = "convert_numeric_with_shellexpand")] @@ -1569,7 +1818,7 @@ pub struct RedisSpec { /// large objects to the redis server. A good rule of thumb is to /// think of the data as: /// `AVAIL_MEMORY / (read_chunk_size * max_chunk_uploads_per_update) = THORETICAL_MAX_CONCURRENT_UPLOADS` - /// (note: it is a good idea to divide `AVAIL_MAX_MEMORY` by ~10 to account for other memory usage) + /// (note: it's a good idea to divide `AVAIL_MAX_MEMORY` by ~10 to account for other memory usage) /// /// Default: 10 #[serde(default, deserialize_with = "convert_numeric_with_shellexpand")] @@ -1663,8 +1912,8 @@ pub struct Retry { #[serde(default)] pub jitter: f32, - /// A list of error codes to retry on, if this is not set then the default - /// error codes to retry on are used. These default codes are the most + /// A list of error codes to retry on, if this isn't set then the default + /// error codes to retry on are used. These default codes are the most /// likely to be non-permanent. /// - `Unknown` /// - `Cancelled` @@ -1775,3 +2024,53 @@ impl Retry { } } } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn zstd_compression_algorithm_parses_and_defaults() { + let cfg: StoreSpec = serde_json5::from_str( + r#"{ compression: { compression_algorithm: { zstd: { + temp_path: "/var/tmp/nl-zstd", max_compressed_upload_size: "512MiB", + compression_level: 19, max_recompression_size: "64MiB" } }, + backend: { memory: {} } } }"#, + ) + .unwrap(); + let StoreSpec::Compression(spec) = cfg else { + panic!("wrong variant") + }; + let CompressionAlgorithm::Zstd(spec) = spec.compression_algorithm else { + panic!("wrong compression algorithm") + }; + assert_eq!(spec.max_compressed_upload_size, 512 * 1024 * 1024); + assert_eq!(spec.compression_level, Some(19)); + assert_eq!(spec.max_recompression_size, 64 * 1024 * 1024); + assert_eq!(spec.max_concurrent_staged_uploads, 0); // 0 = "use default 4" at construction + assert_eq!(spec.max_concurrent_recompressions, 0); // 0 = "use default 1" + assert_eq!(spec.max_concurrent_identity_ops, 0); // 0 = "use default 256" + assert_eq!(spec.stage_timeout_s, 0); // 0 = "use default 600" + assert_eq!(spec.max_inline_commit_size, 0); // 0 = "use default 4MiB" + } + + #[test] + fn zstd_compression_level_may_be_omitted() { + // `compression_level` has no `#[serde(default)]`; confirm the documented + // "omitted means the default level" spelling actually deserializes. + let cfg: StoreSpec = serde_json5::from_str( + r#"{ compression: { compression_algorithm: { zstd: { + temp_path: "/var/tmp/nl-zstd", max_compressed_upload_size: "512MiB" } }, + backend: { memory: {} } } }"#, + ) + .unwrap(); + let StoreSpec::Compression(spec) = cfg else { + panic!("wrong variant") + }; + let CompressionAlgorithm::Zstd(spec) = spec.compression_algorithm else { + panic!("wrong compression algorithm") + }; + assert_eq!(spec.compression_level, None); + assert_eq!(spec.max_recompression_size, 0); + } +} diff --git a/nativelink-config/tests/deserialization_test.rs b/nativelink-config/tests/deserialization_test.rs index 8d476ae24..48a79d308 100644 --- a/nativelink-config/tests/deserialization_test.rs +++ b/nativelink-config/tests/deserialization_test.rs @@ -382,6 +382,8 @@ mod optional_values_tests { } mod boolean_tests { + use nativelink_config::stores::GrpcSpec; + use crate::BoolEntity; #[test] @@ -404,6 +406,32 @@ mod boolean_tests { assert_eq!(deserialized.value, expected, "{input}"); } } + + #[test] + fn grpc_compression_accepts_string_and_environment_boolean() { + let string_spec: GrpcSpec = serde_json5::from_str( + r#"{ + endpoints: [{ address: "http://localhost:1234" }], + store_type: "cas", + experimental_remote_cache_compression: "true" + }"#, + ) + .expect("string boolean should deserialize"); + assert!(string_spec.experimental_remote_cache_compression); + + // Safety: this test uses a unique variable and does not run code that + // reads mutable environment state concurrently. + unsafe { std::env::set_var("TEST_GRPC_REMOTE_CACHE_COMPRESSION", "false") }; + let environment_spec: GrpcSpec = serde_json5::from_str( + r#"{ + endpoints: [{ address: "http://localhost:1234" }], + store_type: "cas", + experimental_remote_cache_compression: "${TEST_GRPC_REMOTE_CACHE_COMPRESSION}" + }"#, + ) + .expect("environment boolean should deserialize"); + assert!(!environment_spec.experimental_remote_cache_compression); + } } mod shellexpand_tests { diff --git a/nativelink-config/tests/json5_test.rs b/nativelink-config/tests/json5_test.rs index 24dbeaa3a..6ce5ed1b3 100644 --- a/nativelink-config/tests/json5_test.rs +++ b/nativelink-config/tests/json5_test.rs @@ -1,24 +1,14 @@ use std::fs; -use std::path::Path; +use std::path::{Path, PathBuf}; use nativelink_config::cas_server::CasConfig; -#[test] -fn test_example_parsing() { - let mut examples_path = Path::new(".") - .canonicalize() - .expect("Can canonicalize current dir"); - - if examples_path.join("nativelink-config").exists() { - // inside bazel - examples_path = examples_path.join("nativelink-config"); - } - examples_path = examples_path.join("examples"); - - let mut found_at_least_one_entry = false; - - for entry in fs::read_dir(&examples_path) - .unwrap_or_else(|e| panic!("Failed to read from {:?}: {}", &examples_path, e)) +/// Parses every `.json5` config directly under `dir`, returning how many it +/// found. Panics with the offending path on the first parse failure. +fn parse_all_configs_in(dir: &Path) -> usize { + let mut parsed = 0; + for entry in + fs::read_dir(dir).unwrap_or_else(|e| panic!("Failed to read from {}: {e}", dir.display())) { let config_file = entry.unwrap().path().display().to_string(); if !config_file.contains(".json5") { @@ -26,8 +16,60 @@ fn test_example_parsing() { } CasConfig::try_from_json5_file(&config_file) .unwrap_or_else(|e| panic!("Error while reading {config_file}: {e}")); - found_at_least_one_entry = true; + parsed += 1; + } + parsed +} + +/// The repository root, whether the test runs from the workspace root or from +/// inside the `nativelink-config` package (as it does under bazel). +fn repo_root() -> PathBuf { + let cwd = Path::new(".") + .canonicalize() + .expect("Can canonicalize current dir"); + if cwd.join("nativelink-config").exists() { + cwd + } else { + cwd.parent() + .expect("nativelink-config has a parent directory") + .to_path_buf() + } +} + +#[test] +fn test_example_parsing() { + let examples_path = repo_root().join("nativelink-config").join("examples"); + assert!( + parse_all_configs_in(&examples_path) > 0, + "expected at least one example config in {}", + examples_path.display() + ); +} + +/// The `deployment-examples` trees are what operators copy, so a config that +/// does not even parse is worse than a missing one. Nothing used to check them, +/// which let an unbalanced brace ship in `local-storage-cas-zstd.json5`. +#[test] +fn test_deployment_example_parsing() { + let deployment_examples = repo_root().join("deployment-examples"); + if !deployment_examples.exists() { + // Not present in the bazel runfiles for this target; the cargo run covers it. + return; + } + + let mut parsed = 0; + for entry in fs::read_dir(&deployment_examples) + .unwrap_or_else(|e| panic!("Failed to read from {}: {e}", deployment_examples.display())) + { + let dir = entry.unwrap().path(); + if dir.is_dir() { + parsed += parse_all_configs_in(&dir); + } } - assert!(found_at_least_one_entry); + assert!( + parsed > 0, + "expected at least one deployment example config under {}", + deployment_examples.display() + ); } diff --git a/nativelink-error/Cargo.toml b/nativelink-error/Cargo.toml index 42be30079..0f4f123b5 100644 --- a/nativelink-error/Cargo.toml +++ b/nativelink-error/Cargo.toml @@ -8,7 +8,7 @@ autoexamples = false autotests = true edition = "2024" name = "nativelink-error" -version = "1.6.1" +version = "1.6.3" [dependencies] nativelink-metric = { path = "../nativelink-metric" } diff --git a/nativelink-macro/Cargo.toml b/nativelink-macro/Cargo.toml index 455a22ca3..f1cb77ede 100644 --- a/nativelink-macro/Cargo.toml +++ b/nativelink-macro/Cargo.toml @@ -4,7 +4,7 @@ lints.workspace = true [package] edition = "2024" name = "nativelink-macro" -version = "1.6.1" +version = "1.6.3" [lib] proc-macro = true @@ -12,4 +12,9 @@ proc-macro = true [dependencies] proc-macro2 = { version = "1.0.94", default-features = false } quote = { version = "1.0.40", default-features = false } -syn = { version = "2.0.117", default-features = false } +syn = { version = "3.0.0", default-features = false, features = [ + "full", + "parsing", + "printing", + "proc-macro", +] } diff --git a/nativelink-metric/Cargo.toml b/nativelink-metric/Cargo.toml index 9a922cacb..da6c549cb 100644 --- a/nativelink-metric/Cargo.toml +++ b/nativelink-metric/Cargo.toml @@ -4,7 +4,7 @@ lints.workspace = true [package] edition = "2024" name = "nativelink-metric" -version = "1.6.1" +version = "1.6.3" [dependencies] nativelink-metric-macro-derive = { path = "nativelink-metric-macro-derive" } diff --git a/nativelink-metric/nativelink-metric-macro-derive/Cargo.toml b/nativelink-metric/nativelink-metric-macro-derive/Cargo.toml index cabb4b44a..743fc5d79 100644 --- a/nativelink-metric/nativelink-metric-macro-derive/Cargo.toml +++ b/nativelink-metric/nativelink-metric-macro-derive/Cargo.toml @@ -1,7 +1,7 @@ [package] edition = "2024" name = "nativelink-metric-macro-derive" -version = "1.6.1" +version = "1.6.3" [lib] proc-macro = true @@ -9,4 +9,9 @@ proc-macro = true [dependencies] proc-macro2 = { version = "1.0.94", default-features = false } quote = { version = "1.0.40", default-features = false } -syn = { version = "2.0.117", default-features = false } +syn = { version = "3.0.0", default-features = false, features = [ + "derive", + "parsing", + "printing", + "proc-macro", +] } diff --git a/nativelink-metric/nativelink-metric-macro-derive/src/lib.rs b/nativelink-metric/nativelink-metric-macro-derive/src/lib.rs index 4488e4a87..b0e1aa969 100644 --- a/nativelink-metric/nativelink-metric-macro-derive/src/lib.rs +++ b/nativelink-metric/nativelink-metric-macro-derive/src/lib.rs @@ -93,7 +93,6 @@ impl ToTokens for MetricKind { } /// Holds general information about a specific field that is to be published. -#[derive(Debug)] struct MetricFieldMetaData<'a> { field_name: &'a Ident, metric_kind: MetricKind, @@ -133,7 +132,6 @@ impl<'a> MetricFieldMetaData<'a> { /// Holds the template information about the struct. This is needed /// to create the `MetricsComponent` impl. -#[derive(Debug)] struct Generics<'a> { implementation: ImplGenerics<'a>, ty: TypeGenerics<'a>, @@ -142,7 +140,6 @@ struct Generics<'a> { /// Holds metadata about the struct that is having `MetricsComponent` /// implemented. -#[derive(Debug)] struct MetricStruct<'a> { name: &'a Ident, metric_fields: Vec>, diff --git a/nativelink-proto/Cargo.toml b/nativelink-proto/Cargo.toml index bf8131fb3..59c46fb03 100644 --- a/nativelink-proto/Cargo.toml +++ b/nativelink-proto/Cargo.toml @@ -2,7 +2,7 @@ [package] edition = "2024" name = "nativelink-proto" -version = "1.6.1" +version = "1.6.3" [lib] doctest = false # because some of the generated protos have things that look like doctests but break diff --git a/nativelink-redis-tester/Cargo.toml b/nativelink-redis-tester/Cargo.toml index f2fcfc665..c859ca559 100644 --- a/nativelink-redis-tester/Cargo.toml +++ b/nativelink-redis-tester/Cargo.toml @@ -4,7 +4,7 @@ lints.workspace = true [package] edition = "2024" name = "nativelink-redis-tester" -version = "1.6.1" +version = "1.6.3" [dependencies] nativelink-util = { path = "../nativelink-util" } diff --git a/nativelink-scheduler/Cargo.toml b/nativelink-scheduler/Cargo.toml index 204c18e6a..4922a28f2 100644 --- a/nativelink-scheduler/Cargo.toml +++ b/nativelink-scheduler/Cargo.toml @@ -4,7 +4,7 @@ lints.workspace = true [package] edition = "2024" name = "nativelink-scheduler" -version = "1.6.1" +version = "1.6.3" [dependencies] nativelink-config = { path = "../nativelink-config" } @@ -59,6 +59,9 @@ nativelink-redis-tester = { path = "../nativelink-redis-tester" } pretty_assertions = { version = "1.4.1", features = [ "std", ], default-features = false } +tokio = { version = "1.52.2", features = [ + "test-util", +], default-features = false } tracing-test = { version = "0.2.5", default-features = false, features = [ "no-env-filter", ] } diff --git a/nativelink-scheduler/src/simple_scheduler.rs b/nativelink-scheduler/src/simple_scheduler.rs index 0abd6bad7..97da8fe1c 100644 --- a/nativelink-scheduler/src/simple_scheduler.rs +++ b/nativelink-scheduler/src/simple_scheduler.rs @@ -532,6 +532,12 @@ impl SimpleScheduler { let worker_scheduler_clone = worker_scheduler.clone(); + let fallback_match_interval = match spec.fallback_match_interval_s { + // Zero or any negative value means disabled. + ..=0 => None, + secs => Some(Duration::from_secs(secs.unsigned_abs())), + }; + let action_scheduler = Arc::new_cyclic(move |weak_self| -> Self { let weak_inner = weak_self.clone(); let task_worker_matching_spawn = @@ -546,16 +552,28 @@ impl SimpleScheduler { tokio::pin!(worker_change_fut); // Wait for either of these futures to be ready. let state_changed = future::select(task_change_fut, worker_change_fut); - if last_match_successful { - let _ = state_changed.await; + let max_wait = if last_match_successful { + // Even on success, periodically re-run the match as a + // fallback for missed notifications and eventually + // consistent backends (e.g. a re-queued operation that + // was not yet visible to the search triggered by its + // own notification). Without this, such an operation + // can stay queued until an unrelated event triggers + // another matching pass. + fallback_match_interval } else { // If the last match failed, then run again after a short sleep. // This resolves issues where we tried to re-schedule a job to // a disconnected worker. The sleep ensures we don't enter a // hard loop if there's something wrong inside do_try_match. - let sleep_fut = tokio::time::sleep(Duration::from_millis(100)); + Some(Duration::from_millis(100)) + }; + if let Some(max_wait) = max_wait { + let sleep_fut = tokio::time::sleep(max_wait); tokio::pin!(sleep_fut); let _ = future::select(state_changed, sleep_fut).await; + } else { + let _ = state_changed.await; } let result = match weak_inner.upgrade() { diff --git a/nativelink-scheduler/src/simple_scheduler_state_manager.rs b/nativelink-scheduler/src/simple_scheduler_state_manager.rs index 0e210ea92..cb487de4b 100644 --- a/nativelink-scheduler/src/simple_scheduler_state_manager.rs +++ b/nativelink-scheduler/src/simple_scheduler_state_manager.rs @@ -747,6 +747,7 @@ where } } + let mut is_retry = false; let stage = match &update { UpdateOperationType::KeepAlive => { awaited_action.worker_keep_alive((self.now_fn)().now()); @@ -795,6 +796,7 @@ where ..ActionResult::default() }) } else { + is_retry = true; ActionStage::Queued } } @@ -826,6 +828,7 @@ where ..ActionResult::default() }) } else { + is_retry = true; ActionStage::Queued } } @@ -920,6 +923,12 @@ where }; attrs.push(KeyValue::new(EXECUTION_RESULT, result)); EXECUTION_METRICS.execution_completed_count.add(1, &attrs); + nativelink_util::metrics::record_completed_execution_metrics( + action_result, + instance_name, + worker_id.as_deref(), + priority, + ); } ActionStage::CompletedFromCache(_) => { attrs.push(KeyValue::new(EXECUTION_RESULT, ExecutionResult::CacheHit)); @@ -928,6 +937,16 @@ where _ => {} } + // A failed attempt that re-queued the action counts as a retry. + if is_retry { + let retry_attrs = nativelink_util::metrics::make_execution_attributes( + instance_name, + worker_id.as_deref(), + priority, + ); + EXECUTION_METRICS.execution_retry_count.add(1, &retry_attrs); + } + debug!( %operation_id, retry_count, diff --git a/nativelink-scheduler/src/worker_capability_index.rs b/nativelink-scheduler/src/worker_capability_index.rs index b0e45b76b..1453a1217 100644 --- a/nativelink-scheduler/src/worker_capability_index.rs +++ b/nativelink-scheduler/src/worker_capability_index.rs @@ -91,7 +91,7 @@ impl WorkerCapabilityIndex { .insert(worker_id.clone()); } PlatformPropertyValue::Minimum(_) | PlatformPropertyValue::Ignore(_) => { - // Minimum properties are tracked via property_presence only. + // Minimum properties are tracked via `property_presence` only. // Their actual values are checked at runtime since they're dynamic. // Ignore properties we just drop diff --git a/nativelink-scheduler/tests/redis_store_awaited_action_db_test.rs b/nativelink-scheduler/tests/redis_store_awaited_action_db_test.rs index 0fd6e4c94..be75aed43 100644 --- a/nativelink-scheduler/tests/redis_store_awaited_action_db_test.rs +++ b/nativelink-scheduler/tests/redis_store_awaited_action_db_test.rs @@ -116,7 +116,7 @@ fn make_awaited_action(operation_id: &str) -> AwaitedAction { ) } -// TODO: This test needs to be rewritten to use workers (like test_multiple_clients_subscribe_to_same_action). +// TODO: This test needs to be rewritten to use workers (like `test_multiple_clients_subscribe_to_same_action`). #[nativelink_test] #[ignore = "needs rewrite to use workers (like test_multiple_clients_subscribe_to_same_action)"] async fn add_action_smoke_test() -> Result<(), Error> { @@ -288,7 +288,7 @@ async fn test_multiple_clients_subscribe_to_same_action() -> Result<(), Error> { .await .unwrap(); - // Second client should be able to get the action by its client_operation_id + // Second client should be able to get the action by its `client_operation_id` let get_subscription = scheduler .filter_operations(OperationFilter { client_operation_id: Some(OperationId::from(CLIENT_OPERATION_ID_2)), @@ -471,7 +471,7 @@ async fn test_orphaned_client_operation_id_returns_none() -> Result<(), Error> { fake_redis_backend.set_subscription_manager(store.subscription_manager().await.unwrap()); // Manually set up the orphaned state in the fake backend: - // 1. Add client_id → operation_id mapping (cid_* key) + // 1. Add `client_id` → `operation_id` mapping (cid_* key) { let mut table = fake_redis_backend.table.lock().unwrap(); let mut client_fields = HashMap::new(); diff --git a/nativelink-scheduler/tests/simple_scheduler_test.rs b/nativelink-scheduler/tests/simple_scheduler_test.rs index 2938304d1..3b6bf7766 100644 --- a/nativelink-scheduler/tests/simple_scheduler_test.rs +++ b/nativelink-scheduler/tests/simple_scheduler_test.rs @@ -2789,3 +2789,225 @@ async fn logs_when_no_workers_match() -> Result<(), Error> { Ok(()) } + +/// Wraps a real `AwaitedActionDb`, but hides queued actions from +/// `get_range_of_actions` while `suppress_queued_searches` is set. This +/// simulates an eventually consistent backend (e.g. Redis), where a +/// (re-)queued operation may not yet be visible to the search that its own +/// change notification triggered. +#[derive(MetricsComponent)] +struct QueuedSearchSuppressingDb { + inner: A, + suppress_queued_searches: Arc, +} + +impl AwaitedActionDb for QueuedSearchSuppressingDb { + type Subscriber = A::Subscriber; + + async fn get_awaited_action_by_id( + &self, + client_operation_id: &OperationId, + ) -> Result, Error> { + self.inner + .get_awaited_action_by_id(client_operation_id) + .await + } + + async fn get_all_awaited_actions( + &self, + ) -> Result> + Send, Error> { + self.inner.get_all_awaited_actions().await + } + + async fn get_by_operation_id( + &self, + operation_id: &OperationId, + ) -> Result, Error> { + self.inner.get_by_operation_id(operation_id).await + } + + async fn get_range_of_actions( + &self, + state: SortedAwaitedActionState, + start: Bound, + end: Bound, + desc: bool, + ) -> Result> + Send, Error> { + let items = if matches!(state, SortedAwaitedActionState::Queued) + && self.suppress_queued_searches.load(Ordering::Acquire) + { + Vec::new() + } else { + self.inner + .get_range_of_actions(state, start, end, desc) + .await? + .collect::>() + .await + }; + Ok(futures::stream::iter(items)) + } + + async fn update_awaited_action(&self, new_awaited_action: AwaitedAction) -> Result<(), Error> { + self.inner.update_awaited_action(new_awaited_action).await + } + + async fn add_action( + &self, + client_operation_id: OperationId, + action_info: Arc, + no_event_action_timeout: Duration, + ) -> Result { + self.inner + .add_action(client_operation_id, action_info, no_event_action_timeout) + .await + } +} + +/// Common setup for the fallback match interval tests: a scheduler over a +/// `QueuedSearchSuppressingDb` with a channel that receives a message after +/// every completed matching pass. +type FallbackTestSetup = ( + Arc, + Arc, + Arc, + mpsc::UnboundedReceiver<()>, +); + +fn make_fallback_test_scheduler(fallback_match_interval_s: i64) -> FallbackTestSetup { + let task_change_notify = Arc::new(Notify::new()); + let suppress_queued_searches = Arc::new(AtomicBool::new(false)); + let (match_tx, match_rx) = mpsc::unbounded_channel(); + let (scheduler, _worker_scheduler) = SimpleScheduler::new_with_callback( + &SimpleSpec { + fallback_match_interval_s, + ..Default::default() + }, + QueuedSearchSuppressingDb { + inner: memory_awaited_action_db_factory( + 0, + &task_change_notify.clone(), + MockInstantWrapped::default, + ), + suppress_queued_searches: suppress_queued_searches.clone(), + }, + move || { + let match_tx = match_tx.clone(); + async move { + let _ = match_tx.send(()); + } + }, + task_change_notify.clone(), + MockInstantWrapped::default, + None, + ); + ( + scheduler, + task_change_notify, + suppress_queued_searches, + match_rx, + ) +} + +/// Waits until no matching pass has completed for a short while, which +/// guarantees no notification permits are pending and no pass is in flight. +async fn wait_for_matching_passes_to_settle(match_rx: &mut mpsc::UnboundedReceiver<()>) { + while tokio::time::timeout(Duration::from_millis(200), match_rx.recv()) + .await + .is_ok() + {} +} + +// Regression test for a queued operation not being visible to the matching +// engine search that its own change notification triggered (e.g. an OOM-killed +// worker's operation being re-queued while the Redis search index is stale). +// The fallback match interval must rescue such an operation. +#[nativelink_test(start_paused = true)] +async fn fallback_match_interval_rescues_action_hidden_from_search() -> Result<(), Error> { + let worker_id = WorkerId("worker_id".to_string()); + let (scheduler, _task_change_notify, suppress_queued_searches, mut match_rx) = + make_fallback_test_scheduler(1 /* fallback_match_interval_s */); + let action_digest = DigestInfo::new([99u8; 32], 512); + + let mut rx_from_worker = + setup_new_worker(&scheduler, worker_id.clone(), PlatformProperties::default()).await?; + + // Hide the action from the matching engine's queued searches, then add it. + suppress_queued_searches.store(true, Ordering::Release); + let insert_timestamp = make_system_time(1); + let mut action_listener = + setup_action(&scheduler, action_digest, HashMap::new(), insert_timestamp).await?; + + wait_for_matching_passes_to_settle(&mut match_rx).await; + + // All notification-triggered passes ran while the action was hidden, so + // nothing was assigned to the worker. + assert_eq!(poll!(Box::pin(rx_from_worker.recv())), Poll::Pending); + + // Make the action visible again. No notification fires for this, so only + // the fallback match interval can rescue the action now. + suppress_queued_searches.store(false, Ordering::Release); + + let msg_for_worker = tokio::time::timeout(Duration::from_secs(30), rx_from_worker.recv()) + .await + .expect("Fallback matching pass should have assigned the action") + .unwrap(); + match msg_for_worker.update { + Some(update_for_worker::Update::StartAction(start_execute)) => { + assert_eq!( + start_execute.execute_request.unwrap().action_digest, + Some(action_digest.into()) + ); + } + other => panic!("Expected StartAction, got: {other:?}"), + } + + // Client should see the action executing. + let (action_state, _maybe_origin_metadata) = action_listener.changed().await.unwrap(); + assert_eq!(action_state.stage, ActionStage::Executing); + + Ok(()) +} + +// With the fallback match interval disabled, the same scenario leaves the +// action stuck in the queued state until an unrelated event triggers another +// matching pass. This documents the behavior the fallback interval fixes. +#[nativelink_test(start_paused = true)] +async fn fallback_match_interval_disabled_leaves_hidden_action_queued() -> Result<(), Error> { + let worker_id = WorkerId("worker_id".to_string()); + let (scheduler, task_change_notify, suppress_queued_searches, mut match_rx) = + make_fallback_test_scheduler(-1 /* fallback_match_interval_s */); + let action_digest = DigestInfo::new([99u8; 32], 512); + + let mut rx_from_worker = + setup_new_worker(&scheduler, worker_id.clone(), PlatformProperties::default()).await?; + + // Hide the action from the matching engine's queued searches, then add it. + suppress_queued_searches.store(true, Ordering::Release); + let insert_timestamp = make_system_time(1); + let _action_listener = + setup_action(&scheduler, action_digest, HashMap::new(), insert_timestamp).await?; + + wait_for_matching_passes_to_settle(&mut match_rx).await; + suppress_queued_searches.store(false, Ordering::Release); + + // Without the fallback interval nothing ever rescues the action. + assert!( + tokio::time::timeout(Duration::from_secs(30), rx_from_worker.recv()) + .await + .is_err(), + "Action should stay queued with the fallback match interval disabled" + ); + + // Only an unrelated task change event triggers another matching pass. + task_change_notify.notify_one(); + let msg_for_worker = tokio::time::timeout(Duration::from_secs(5), rx_from_worker.recv()) + .await + .expect("Task change notification should have assigned the action") + .unwrap(); + assert!(matches!( + msg_for_worker.update, + Some(update_for_worker::Update::StartAction(_)) + )); + + Ok(()) +} diff --git a/nativelink-service/BUILD.bazel b/nativelink-service/BUILD.bazel index 43aa6d168..72b9cbd8f 100644 --- a/nativelink-service/BUILD.bazel +++ b/nativelink-service/BUILD.bazel @@ -69,6 +69,7 @@ rust_test_suite( "tests/execution_server_test.rs", "tests/fastcdc_conformance_test.rs", "tests/fetch_server_test.rs", + "tests/grpc_wire_compression_test.rs", "tests/health_server_test.rs", "tests/push_server_test.rs", "tests/wire_compression_test.rs", diff --git a/nativelink-service/Cargo.toml b/nativelink-service/Cargo.toml index be34c0102..2a2d1e5b6 100644 --- a/nativelink-service/Cargo.toml +++ b/nativelink-service/Cargo.toml @@ -4,7 +4,7 @@ lints.workspace = true [package] edition = "2024" name = "nativelink-service" -version = "1.6.1" +version = "1.6.3" [dependencies] nativelink-config = { path = "../nativelink-config" } diff --git a/nativelink-service/src/bytestream_server.rs b/nativelink-service/src/bytestream_server.rs index 026a511e0..3acb6c4ea 100644 --- a/nativelink-service/src/bytestream_server.rs +++ b/nativelink-service/src/bytestream_server.rs @@ -17,8 +17,8 @@ use core::fmt::{Debug, Formatter}; use core::pin::Pin; use core::sync::atomic::{AtomicU64, Ordering}; use core::time::Duration; -use std::collections::HashMap; use std::collections::hash_map::Entry; +use std::collections::{HashMap, HashSet}; use std::sync::Arc; use std::time::{Instant, SystemTime, UNIX_EPOCH}; @@ -50,16 +50,16 @@ use nativelink_util::digest_hasher::{ }; use nativelink_util::proto_stream_utils::WriteRequestStreamWrapper; use nativelink_util::resource_info::ResourceInfo; +use nativelink_util::spawn; use nativelink_util::store_trait::{Store, StoreLike, StoreOptimizations, UploadSizeInfo}; use nativelink_util::task::JoinHandleDropGuard; -use nativelink_util::{spawn, spawn_blocking}; use opentelemetry::context::FutureExt; use parking_lot::Mutex; use tokio::time::sleep; use tonic::{Request, Response, Status, Streaming}; use tracing::{Instrument, Level, debug, error, error_span, info, instrument, trace, warn}; -use crate::wire_compression::RemoteCacheCompressionInstances; +use crate::wire_compression::{RemoteCacheCompressionInstances, wire_compressor_capability}; /// If this value changes update the documentation in the config definition. const DEFAULT_PERSIST_STREAM_ON_DISCONNECT_TIMEOUT: Duration = Duration::from_mins(1); @@ -67,6 +67,9 @@ const DEFAULT_PERSIST_STREAM_ON_DISCONNECT_TIMEOUT: Duration = Duration::from_mi /// If this value changes update the documentation in the config definition. const DEFAULT_MAX_BYTES_PER_STREAM: usize = 64 * 1024; +/// If this value changes update the documentation in the config definition. +const DEFAULT_COMPRESSED_UPLOAD_IDLE_TIMEOUT: Duration = Duration::from_mins(1); + /// Metrics for `ByteStream` server operations. /// Tracks upload/download activity, throughput, and latency. #[derive(Debug, Default)] @@ -254,6 +257,10 @@ pub struct InstanceInfo { active_uploads: Arc>>, /// How long to keep idle streams before timing them out. idle_stream_timeout: Duration, + /// How long to wait for the next `WriteRequest` of a compressed upload. + /// Distinct from `idle_stream_timeout`, which governs how long a + /// disconnected upload stays resumable. + compressed_upload_idle_timeout: Duration, metrics: Arc, /// Handle to the global sweeper task. Kept alive for the lifetime of the instance. _sweeper_handle: Arc>, @@ -268,6 +275,10 @@ impl Debug for InstanceInfo { .field("max_bytes_per_stream", &self.max_bytes_per_stream) .field("active_uploads", &self.active_uploads) .field("idle_stream_timeout", &self.idle_stream_timeout) + .field( + "compressed_upload_idle_timeout", + &self.compressed_upload_idle_timeout, + ) .field("metrics", &self.metrics) .field( "remote_cache_compression_enabled", @@ -337,6 +348,34 @@ impl InstanceInfo { } } +/// Returns the first client-attributable error among `results`: a bad request or +/// an expired deadline. Those are root causes worth reporting verbatim, whereas +/// the other halves of a joined pipeline typically fail with a consequential +/// "channel disconnected" error that would mask them. +fn first_client_attributable_error<'a>( + results: impl IntoIterator>, +) -> Option { + results + .into_iter() + .flatten() + .find(|err| { + matches!( + err.code, + Code::InvalidArgument | Code::DeadlineExceeded | Code::ResourceExhausted + ) + }) + .cloned() +} + +/// Folds `err` into an accumulated upload error, keeping the earlier error's +/// messages first. +fn merge_upload_error(accumulated: Option, err: Error) -> Error { + match accumulated { + Some(existing) => existing.merge(err), + None => err, + } +} + /// Pump compressed `ByteStream` upload chunks into the decoder. /// /// Compressed uploads intentionally do not support the identity upload resume @@ -345,9 +384,25 @@ async fn process_compressed_client_stream( mut stream: WriteRequestStreamWrapper> + Unpin>, mut tx: DropCloserWriteHalf, bytes_received: &Arc, + idle_timeout: Duration, ) -> Result<(), Error> { loop { - match stream.next().await { + // Bounds the wait for the *next* `WriteRequest` only, so continuous + // progress never trips it. A client that trickles bytes to hold a store + // slot open is bounded separately, by the store's own total staging + // deadline. Dropping `tx` on return disconnects the compressed channel, + // unwinding the store-side task through the caller's `join`. + let next = match tokio::time::timeout(idle_timeout, stream.next()).await { + Ok(next) => next, + Err(_elapsed) => { + return Err(make_err!( + Code::DeadlineExceeded, + "Compressed upload idle timeout ({}s) elapsed waiting for the next WriteRequest", + idle_timeout.as_secs() + )); + } + }; + match next { Some(Ok(write_request)) => { if write_request.write_offset < 0 { return Err(make_input_err!( @@ -398,6 +453,7 @@ struct StreamState { uuid: UuidKey, tx: DropCloserWriteHalf, store_update_fut: StoreUpdateFuture, + digest_function: DigestHasherFunc, } impl Debug for StreamState { @@ -481,6 +537,7 @@ impl IdleStream { #[derive(Debug)] pub struct ByteStreamServer { instance_infos: HashMap, + reported_digest_function_mismatches: Mutex>, } impl ByteStreamServer { @@ -519,7 +576,10 @@ impl ByteStreamServer { )?, ); } - Ok(Self { instance_infos }) + Ok(Self { + instance_infos, + reported_digest_function_mismatches: Mutex::new(HashSet::new()), + }) } pub fn new_with_timeout( @@ -536,6 +596,11 @@ impl ByteStreamServer { } else { config.max_bytes_per_stream }; + let compressed_upload_idle_timeout = if config.compressed_upload_idle_timeout_s == 0 { + DEFAULT_COMPRESSED_UPLOAD_IDLE_TIMEOUT + } else { + Duration::from_secs(config.compressed_upload_idle_timeout_s as u64) + }; let active_uploads: Arc>> = Arc::new(Mutex::new(HashMap::new())); @@ -597,6 +662,7 @@ impl ByteStreamServer { max_bytes_per_stream, active_uploads, idle_stream_timeout, + compressed_upload_idle_timeout, metrics, _sweeper_handle: Arc::new(sweeper_handle), remote_cache_compression_enabled, @@ -623,7 +689,13 @@ impl ByteStreamServer { uuid_str: &str, instance: &InstanceInfo, digest: DigestInfo, - ) -> ActiveStreamGuard { + digest_function: DigestHasherFunc, + ) -> Result { + // Bind the digest function to the retained store future itself. The + // future can outlive this RPC while an upload is idle, so it must not + // depend on the context of whichever request polls it next. + let store_update_context = make_ctx_for_hash_func(digest_function)?; + // Parse UUID string to u128 key for efficient HashMap operations let uuid_key = parse_uuid_to_key(uuid_str); @@ -632,6 +704,15 @@ impl ByteStreamServer { match active_uploads.entry(uuid_key) { Entry::Occupied(mut entry) => { let maybe_idle_stream = entry.get_mut(); + if let Some(idle_stream) = maybe_idle_stream.1.as_ref() + && idle_stream.stream_state.digest_function != digest_function + { + return Err(make_input_err!( + "Cannot resume upload with digest function {} because it started with {}", + digest_function, + idle_stream.stream_state.digest_function, + )); + } if let Some(idle_stream) = maybe_idle_stream.1.take() { // Case 2: Stream exists but is idle, we can resume it let bytes_received = maybe_idle_stream.0.clone(); @@ -644,7 +725,7 @@ impl ByteStreamServer { .metrics .resumed_uploads .fetch_add(1, Ordering::Relaxed); - return idle_stream.into_active_stream(bytes_received, instance); + return Ok(idle_stream.into_active_stream(bytes_received, instance)); } // Case 3: Stream is active - generate a unique UUID to avoid collision. // Using nanosecond timestamp makes collision probability essentially zero. @@ -690,25 +771,29 @@ impl ByteStreamServer { let (tx, rx) = make_buf_channel_pair(); let store = instance.store.clone(); - let store_update_fut = Box::pin(async move { - // We need to wrap `Store::update()` in a another future because we need to capture - // `store` to ensure its lifetime follows the future and not the caller. - store - // Bytestream always uses digest size as the actual byte size. - .update(digest, rx, UploadSizeInfo::ExactSize(digest.size_bytes())) - .await - .map(|_| ()) - }); - ActiveStreamGuard { + let store_update_fut = Box::pin( + async move { + // We need to wrap `Store::update()` in a another future because we need to capture + // `store` to ensure its lifetime follows the future and not the caller. + store + // Bytestream always uses digest size as the actual byte size. + .update(digest, rx, UploadSizeInfo::ExactSize(digest.size_bytes())) + .await + .map(|_| ()) + } + .with_context(store_update_context), + ); + Ok(ActiveStreamGuard { stream_state: Some(StreamState { uuid, tx, store_update_fut, + digest_function, }), bytes_received, active_uploads: instance.active_uploads.clone(), metrics: instance.metrics.clone(), - } + }) } async fn inner_read( @@ -840,6 +925,7 @@ impl ByteStreamServer { &self, instance_info: &InstanceInfo, digest: DigestInfo, + digest_function: DigestHasherFunc, stream: WriteRequestStreamWrapper> + Unpin>, ) -> Result, Error> { async fn process_client_stream( @@ -935,7 +1021,7 @@ impl ByteStreamServer { .as_ref() .ok_or_else(|| make_input_err!("UUID must be set if writing data"))?; let mut active_stream_guard = - self.create_or_join_upload_stream(uuid, instance_info, digest); + self.create_or_join_upload_stream(uuid, instance_info, digest, digest_function)?; let expected_size = stream.resource_info.expected_size as u64; let active_stream = active_stream_guard.stream_state.as_mut().unwrap(); @@ -954,7 +1040,7 @@ impl ByteStreamServer { active_stream_guard.graceful_finish(); Ok(Response::new(WriteResponse { - committed_size: expected_size as i64, + committed_size: expected_size.try_into().unwrap_or(i64::MAX), })) } @@ -964,6 +1050,7 @@ impl ByteStreamServer { &self, instance_info: &InstanceInfo, digest: DigestInfo, + digest_function: DigestHasherFunc, mut stream: WriteRequestStreamWrapper< impl Stream> + Unpin, >, @@ -1038,15 +1125,17 @@ impl ByteStreamServer { // Direct update without channel overhead let store = instance_info.store.clone(); + let store_update_context = make_ctx_for_hash_func(digest_function)?; store .update_oneshot(digest, buffer.freeze()) + .with_context(store_update_context) .await .err_tip(|| "Error in update_oneshot")?; // Note: bytes_written_total is updated in the caller (bytestream_write) based on result Ok(Response::new(WriteResponse { - committed_size: expected_size as i64, + committed_size: expected_size.try_into().unwrap_or(i64::MAX), })) } @@ -1073,8 +1162,52 @@ impl ByteStreamServer { let (bytes_received, _guard) = instance.track_compressed_upload(uuid_key); let (compressed_tx, compressed_rx) = make_buf_channel_pair(); + + // Fast path: when the immediate instance store can accept the negotiated + // wire representation directly, hand it the client's COMPRESSED stream to + // validate/stage/commit byte-for-byte, skipping the decode + re-encode + // round trip. The client stream still runs through + // `process_compressed_client_stream` so `bytes_received` (and therefore + // QueryWriteStatus and `committed_size`) still tracks compressed + // wire-byte progress. + let maybe_wire_store = wire_compressor_capability(wire_compressor) + .and_then(|capability| Some((instance.store.wire_compression_store()?, capability))); + if let Some((wire_store, capability)) = maybe_wire_store { + let update_fut = + wire_store.update_compressed(digest, digest_function, capability, compressed_rx); + let client_stream_fut = process_compressed_client_stream( + stream, + compressed_tx, + &bytes_received, + instance.compressed_upload_idle_timeout, + ); + let (client_stream_result, update_result) = tokio::join!(client_stream_fut, update_fut); + + if let Some(err) = first_client_attributable_error([ + client_stream_result.as_ref().err(), + update_result.as_ref().err(), + ]) { + return Err(err); + } + let mut upload_error = update_result.err(); + if let Err(err) = client_stream_result { + upload_error = Some(merge_upload_error(upload_error, err)); + } + if let Some(err) = upload_error { + return Err(err); + } + + // `committed_size` stays the compressed wire byte count tracked by + // the atomic, which should agree with the capability's returned wire + // byte count. + let committed_size = i64::try_from(bytes_received.load(Ordering::Acquire)) + .err_tip(|| "Compressed upload size was not convertible to i64")?; + return Ok(Response::new(WriteResponse { committed_size })); + } + let (decompressed_tx, decompressed_rx) = make_buf_channel_pair(); let store = instance.store.clone(); + let store_update_context = make_ctx_for_hash_func(digest_function)?; let store_update_fut = async move { store .update( @@ -1085,47 +1218,39 @@ impl ByteStreamServer { .await .map(|_| ()) .err_tip(|| "Failed to store decompressed data") - }; - let decode_fut = async move { - spawn_blocking!("bytestream_decode_compressed_upload", move || { - crate::wire_compression::stream_decode_compressed_upload( - compressed_rx, - wire_compressor, - digest, - digest_function, - decompressed_tx, - ) - }) - .await - .map_err(|e| make_err!(Code::Internal, "Decompression task failed: {}", e))? - }; - let client_stream_fut = - process_compressed_client_stream(stream, compressed_tx, &bytes_received); + } + .with_context(store_update_context); + // Plain async future: decode progresses at the pace of the client + // upload and the store write without occupying a blocking-pool thread + // for the stream's lifetime. + let decode_fut = crate::wire_compression::stream_decode_compressed_upload( + compressed_rx, + wire_compressor, + digest, + digest_function, + decompressed_tx, + ); + let client_stream_fut = process_compressed_client_stream( + stream, + compressed_tx, + &bytes_received, + instance.compressed_upload_idle_timeout, + ); let (client_stream_result, decode_result, store_update_result) = tokio::join!(client_stream_fut, decode_fut, store_update_fut); - if let Err(err) = &client_stream_result - && err.code == Code::InvalidArgument - { - return Err(err.clone()); - } - if let Err(err) = &decode_result - && err.code == Code::InvalidArgument - { - return Err(err.clone()); + if let Some(err) = first_client_attributable_error([ + client_stream_result.as_ref().err(), + decode_result.as_ref().err(), + ]) { + return Err(err); } let mut upload_error = store_update_result.err(); if let Err(err) = decode_result { - upload_error = Some(match upload_error { - Some(existing) => existing.merge(err), - None => err, - }); + upload_error = Some(merge_upload_error(upload_error, err)); } if let Err(err) = client_stream_result { - upload_error = Some(match upload_error { - Some(existing) => existing.merge(err), - None => err, - }); + upload_error = Some(merge_upload_error(upload_error, err)); } if let Some(err) = upload_error { return Err(err); @@ -1184,6 +1309,8 @@ impl ByteStreamServer { } } + type BoxedResultFuture = Pin> + Send>>; + if read_request.read_limit != 0 { return Err(make_input_err!( "read_limit must be 0 when reading compressed blobs" @@ -1197,31 +1324,63 @@ impl ByteStreamServer { let read_offset = u64::try_from(read_request.read_offset) .err_tip(|| "Could not convert read_offset to u64")?; - let (raw_tx, raw_rx) = make_buf_channel_pair(); - let (compressed_tx, compressed_rx) = make_buf_channel_pair(); + // Fast path: at offset 0, when the immediate instance store already holds + // the negotiated wire representation, pipe its stored stream + // byte-for-byte into the same chunking pipeline, skipping the raw + // `get_part` + re-encode. REAPI defines `read_offset` against the + // *uncompressed* blob, so a resume at offset > 0 (and any instance + // without the capability) keeps the raw read + re-encode path. + let maybe_wire_store = if read_offset == 0 { + wire_compressor_capability(wire_compressor) + .and_then(|capability| Some((instance.store.wire_compression_store()?, capability))) + } else { + None + }; - let store = instance.store.clone(); - let get_part_fut = Box::pin(async move { - store - .get_part(digest, raw_tx, read_offset, None) - .await - .err_tip(|| "Failed to read blob for wire compression") - }); - let encode_fut = Box::pin(async move { - spawn_blocking!("bytestream_encode_compressed_download", move || { - crate::wire_compression::stream_encode_compressed_download( + let (rx, get_part_fut, encode_fut): ( + DropCloserReadHalf, + BoxedResultFuture, + BoxedResultFuture, + ) = if let Some((wire_store, capability)) = maybe_wire_store { + let (zstd_tx, zstd_rx) = make_buf_channel_pair(); + let get_part_fut: BoxedResultFuture = Box::pin(async move { + wire_store + .get_compressed(digest, capability, zstd_tx) + .await + .err_tip(|| "Failed to read stored compressed stream for passthrough") + }); + // No re-encode on the fast path; a ready Ok keeps the existing + // error-merge machinery in `ReaderState::finish` a no-op here. + let encode_fut: BoxedResultFuture = Box::pin(async { Ok(()) }); + (zstd_rx, get_part_fut, encode_fut) + } else { + let (raw_tx, raw_rx) = make_buf_channel_pair(); + let (compressed_tx, compressed_rx) = make_buf_channel_pair(); + let store = instance.store.clone(); + let get_part_fut: BoxedResultFuture = Box::pin(async move { + store + .get_part(digest, raw_tx, read_offset, None) + .await + .err_tip(|| "Failed to read blob for wire compression") + }); + // The encode runs as a plain async future: it must not occupy a + // blocking-pool thread for the stream's lifetime, because it only + // progresses at the client's drain rate. Dropping the returned + // stream drops this future, which tears the encode down exactly + // like the previous task-abort-on-drop did. + let encode_fut: BoxedResultFuture = + Box::pin(crate::wire_compression::stream_encode_compressed_download( raw_rx, wire_compressor, + crate::wire_compression::ZSTD_COMPRESSION_LEVEL, compressed_tx, - ) - }) - .await - .map_err(|e| make_err!(Code::Internal, "Compression task failed: {}", e))? - }); + )); + (compressed_rx, get_part_fut, encode_fut) + }; let state = Some(ReaderState { max_bytes_per_stream: instance.max_bytes_per_stream, - rx: compressed_rx, + rx, maybe_get_part_result: None, maybe_encode_result: None, get_part_fut, @@ -1324,7 +1483,10 @@ impl ByteStreamServer { let active_uploads = instance.active_uploads.lock(); if let Some((received_bytes, _maybe_idle_stream)) = active_uploads.get(&uuid_key) { return Ok(Response::new(QueryWriteStatusResponse { - committed_size: received_bytes.load(Ordering::Acquire) as i64, + committed_size: received_bytes + .load(Ordering::Acquire) + .try_into() + .unwrap_or(i64::MAX), // If we are in the active_uploads map, but the value is None, // it means the stream is not complete. complete: false, @@ -1344,7 +1506,7 @@ impl ByteStreamServer { })); }; Ok(Response::new(QueryWriteStatusResponse { - committed_size: item_size as i64, + committed_size: item_size.try_into().unwrap_or(i64::MAX), complete: true, })) } @@ -1367,7 +1529,8 @@ impl ByteStream for ByteStreamServer { let start_time = Instant::now(); let read_request = grpc_request.into_inner(); - let resource_info = ResourceInfo::new(&read_request.resource_name, false)?; + let resource_name = read_request.resource_name.clone(); + let resource_info = ResourceInfo::new(&resource_name, false)?; let instance_name = resource_info.instance_name.as_ref(); let expected_size = resource_info.expected_size as u64; let instance = self @@ -1375,6 +1538,11 @@ impl ByteStream for ByteStreamServer { .get(instance_name) .err_tip(|| format!("'instance_name' not configured for '{instance_name}'"))?; + trace!( + resource_name, + instance_name, expected_size, "Starting bytestream request" + ); + // Track read request instance .metrics @@ -1427,13 +1595,19 @@ impl ByteStream for ByteStreamServer { }; // Track metrics based on result + let elapsed = start_time.elapsed(); #[allow(clippy::cast_possible_truncation)] - let elapsed_ns = start_time.elapsed().as_nanos() as u64; + let elapsed_ns = elapsed.as_nanos() as u64; instance .metrics .read_duration_ns .fetch_add(elapsed_ns, Ordering::Relaxed); + trace!( + ?elapsed, + resource_name, instance_name, expected_size, "Completed bytestream request" + ); + match &resp { Ok(_) => { instance @@ -1502,14 +1676,26 @@ impl ByteStream for ByteStreamServer { return resp; } - let digest_function = stream - .resource_info - .digest_function - .as_deref() - .map_or_else( - || Ok(default_digest_hasher_func()), - DigestHasherFunc::try_from, - )?; + let default_digest_function = default_digest_hasher_func(); + let digest_function = match stream.resource_info.digest_function.as_deref() { + Some(value) => { + let digest_function = DigestHasherFunc::try_from(value)?; + if digest_function != default_digest_function + && self + .reported_digest_function_mismatches + .lock() + .insert((digest_function, default_digest_function)) + { + warn!( + client_digest_function = %digest_function, + server_default_digest_function = %default_digest_function, + "ByteStream client declared a digest function that differs from the server default; the client-declared function will be used for this upload, but clients using different digest functions generate different cache keys and will not share cache hits; configure global.default_digest_hash_function and all clients to use the same digest function" + ); + } + digest_function + } + None => default_digest_function, + }; // Determine if the client is sending wire-compressed data via compressed-blobs URI. let wire_compressor = crate::wire_compression::resolve_wire_compressor( @@ -1523,10 +1709,6 @@ impl ByteStream for ByteStreamServer { let result = self .inner_write_compressed(instance, digest, digest_function, wire_compressor, stream) .instrument(error_span!("bytestream_write_compressed")) - .with_context( - make_ctx_for_hash_func(digest_function) - .err_tip(|| "In BytestreamServer::write_compressed")?, - ) .await .err_tip(|| "In ByteStreamServer::write_compressed"); @@ -1590,21 +1772,13 @@ impl ByteStream for ByteStreamServer { }; let result = if use_oneshot { - self.inner_write_oneshot(instance, digest, stream) + self.inner_write_oneshot(instance, digest, digest_function, stream) .instrument(error_span!("bytestream_write_oneshot")) - .with_context( - make_ctx_for_hash_func(digest_function) - .err_tip(|| "In BytestreamServer::write")?, - ) .await .err_tip(|| "In ByteStreamServer::write (oneshot)") } else { - self.inner_write(instance, digest, stream) + self.inner_write(instance, digest, digest_function, stream) .instrument(error_span!("bytestream_write")) - .with_context( - make_ctx_for_hash_func(digest_function) - .err_tip(|| "In BytestreamServer::write")?, - ) .await .err_tip(|| "In ByteStreamServer::write") }; diff --git a/nativelink-service/src/cas_server.rs b/nativelink-service/src/cas_server.rs index 953c2bc6e..6d7ad536e 100644 --- a/nativelink-service/src/cas_server.rs +++ b/nativelink-service/src/cas_server.rs @@ -13,6 +13,7 @@ // limitations under the License. use core::convert::Into; +use core::future::Future; use core::pin::{Pin, pin}; use core::sync::atomic::{AtomicBool, AtomicU64, Ordering}; use core::time::Duration; @@ -43,16 +44,18 @@ use nativelink_store::grpc_store::GrpcStore; use nativelink_store::store_manager::StoreManager; use nativelink_util::buf_channel::make_buf_channel_pair; use nativelink_util::common::DigestInfo; -use nativelink_util::digest_hasher::{DigestHasher, DigestHasherFunc, make_ctx_for_hash_func}; +use nativelink_util::digest_hasher::{ + DigestHasher, DigestHasherFunc, digest_hasher_func_from_context, make_ctx_for_hash_func, +}; use nativelink_util::spawn_blocking; -use nativelink_util::store_trait::{Store, StoreLike, UploadSizeInfo}; +use nativelink_util::store_trait::{Store, StoreLike, UploadSizeInfo, WireCompressor}; use opentelemetry::context::FutureExt; use prost::Message; use tokio_util::io::StreamReader; use tonic::{Request, Response, Status}; use tracing::{Instrument, Level, debug, error_span, instrument, warn}; -use crate::wire_compression::RemoteCacheCompressionInstances; +use crate::wire_compression::{RemoteCacheCompressionInstances, wire_compressor_capability}; /// Metrics for the experimental `SplitBlob`/`SpliceBlob` chunking RPCs. /// The split hit rate (`split_hits` / `split_requests_total`) indicates how @@ -183,6 +186,25 @@ type GetTreeStream = Pin> /// Per-blob deadline applied inside `BatchReadBlobs` / `BatchUpdateBlobs`. const BATCH_PER_BLOB_TIMEOUT: Duration = Duration::from_secs(30); +/// Runs one BatchReadBlobs/BatchUpdateBlobs store operation under its individual +/// deadline, preserving the RPC-specific timeout and store-error context. +async fn run_batch_blob_with_timeout( + rpc_name: &str, + digest: DigestInfo, + store_error_context: &'static str, + operation: impl Future>, +) -> Result { + match tokio::time::timeout(BATCH_PER_BLOB_TIMEOUT, operation).await { + Ok(result) => result.err_tip(|| store_error_context), + Err(_elapsed) => Err(make_err!( + Code::DeadlineExceeded, + "{rpc_name} per-blob timeout ({} s) elapsed for digest {}", + BATCH_PER_BLOB_TIMEOUT.as_secs(), + digest, + )), + } +} + /// Maximum size of a single chunk accepted in a `SpliceBlob` request. /// Deliberately looser than the largest chunk the server ever advertises /// (4x the maximum allowed average = 4 MiB) so clients using their own @@ -341,6 +363,17 @@ impl CasServer { let remote_cache_compression_enabled = self .remote_cache_compression_instances .enabled_for(instance_name); + // The digest hasher is set on the ambient context by the + // `batch_update_blobs` wrapper (via `make_ctx_for_hash_func`); resolve + // it once for the whole request so the wire-compression fast path can + // validate the staged blob against the client's declared digest function. + let digest_function = digest_hasher_func_from_context(); + // Detect an immediate wire-compression capability at the instance + // boundary so a client-supplied zstd blob can be passed through + // byte-for-byte (validate + stage + commit) instead of decompressed then + // recompressed. Resolved once and cloned into each per-blob future. + let maybe_wire_store = store.wire_compression_store(); + let maybe_wire_store = &maybe_wire_store; let update_futures: FuturesUnordered<_> = request .requests .into_iter() @@ -351,31 +384,55 @@ impl CasServer { .clone() .err_tip(|| "Digest not found in request")?; let digest_info = DigestInfo::try_from(digest.clone())?; - let size_bytes = usize::try_from(digest_info.size_bytes()) - .err_tip(|| "Digest size_bytes was not convertible to usize")?; - - let store_data = crate::wire_compression::decompress_batch_update( - request.data, - request.compressor, - size_bytes, - remote_cache_compression_enabled, - ) - .await?; + + let request_capability = compressor::Value::try_from(request.compressor) + .ok() + .and_then(wire_compressor_capability); // Apply a per-blob deadline so one slow upload does not // make the whole batch hit the client's overall deadline. - let result = match tokio::time::timeout( - BATCH_PER_BLOB_TIMEOUT, - store_ref.update_oneshot(digest_info, store_data), - ) - .await + // The fast path is only taken when remote cache + // compression is enabled for this instance; otherwise a + // zstd-compressed blob falls through to + // `decompress_batch_update` below, which rejects it the + // same way it would without a wire-compression capability. + let result = if remote_cache_compression_enabled + && let (Some(wire_store), Some(capability)) = + (maybe_wire_store.clone(), request_capability) { - Ok(r) => r.err_tip(|| "Error writing to store"), - Err(_elapsed) => Err(make_err!( - Code::DeadlineExceeded, - "BatchUpdateBlobs per-blob timeout ({} s) elapsed for digest {}", - BATCH_PER_BLOB_TIMEOUT.as_secs(), + // Fast path: hand the compressed bytes straight to the + // wire-capable store, which validates them against the + // digest and stores them verbatim. A malformed stream + // surfaces here as this blob's `InvalidArgument`, isolated + // from siblings. + run_batch_blob_with_timeout( + "BatchUpdateBlobs", digest_info, - )), + "Error writing to store", + wire_store.update_compressed_oneshot( + digest_info, + digest_function, + capability, + request.data, + ), + ) + .await + } else { + let size_bytes = usize::try_from(digest_info.size_bytes()) + .err_tip(|| "Digest size_bytes was not convertible to usize")?; + let store_data = crate::wire_compression::decompress_batch_update( + request.data, + request.compressor, + size_bytes, + remote_cache_compression_enabled, + ) + .await?; + run_batch_blob_with_timeout( + "BatchUpdateBlobs", + digest_info, + "Error writing to store", + store_ref.update_oneshot(digest_info, store_data), + ) + .await }; Ok::<_, Error>(batch_update_blobs_response::Response { digest: Some(digest), @@ -419,6 +476,13 @@ impl CasServer { compressor::Value::try_from(*compressor_i32) .is_ok_and(|compressor| compressor == compressor::Value::Zstd) }); + // Detect an immediate wire-compression capability at the instance + // boundary so stored zstd bytes can be returned byte-for-byte (when the + // client accepts zstd and compression actually helped) with no + // decompress-then-recompress round trip. Resolved once and cloned into + // each per-blob future. + let maybe_wire_store = store.wire_compression_store(); + let maybe_wire_store = &maybe_wire_store; let read_futures: FuturesUnordered<_> = request .digests .into_iter() @@ -428,48 +492,68 @@ impl CasServer { // TODO(palfrey) There is a security risk here of someone taking all the memory on the instance. // Apply a per-blob deadline so one slow read does not // make the whole batch hit the client's overall deadline. - let result = match tokio::time::timeout( - BATCH_PER_BLOB_TIMEOUT, - store_ref.get_part_unchunked(digest_copy, 0, None), - ) - .await - { - Ok(r) => r.err_tip(|| "Error reading from store"), - Err(_elapsed) => Err(make_err!( - Code::DeadlineExceeded, - "BatchReadBlobs per-blob timeout ({} s) elapsed for digest {}", - BATCH_PER_BLOB_TIMEOUT.as_secs(), - digest_copy, - )), - }; - let (status, data, response_compressor) = match result { - Ok(raw_data) => { - let (output_data, chosen_compressor) = if client_accepts_zstd { - let data_for_compression = raw_data.clone(); - match spawn_blocking!("cas_encode_compressed_download", move || { - crate::wire_compression::compress_for_batch_read( - data_for_compression, - ) - }) - .await - { - Ok(compressed_result) => compressed_result, - Err(e) => { - warn!("Wire compression task failed for digest {:?}, falling back to identity: {}", digest, e); - (raw_data, compressor::Value::Identity) + let outcome: Result<(Bytes, compressor::Value), Error> = + if let Some(wire_store) = maybe_wire_store.clone() { + // Fast path: the wire-capable store chooses between its + // stored zstd bytes (when the client accepts zstd and they + // are smaller than the raw size) and the decoded raw bytes, + // avoiding a decompress-then-recompress round trip. + let acceptable_compressors = + client_accepts_zstd.then_some(WireCompressor::Zstd); + run_batch_blob_with_timeout( + "BatchReadBlobs", + digest_copy, + "Error reading from store", + wire_store + .get_for_batch(digest_copy, acceptable_compressors.as_slice()), + ) + .await + .map(|(data, maybe_compressor)| { + let chosen = match maybe_compressor { + Some(WireCompressor::Zstd) => compressor::Value::Zstd, + None => compressor::Value::Identity, + }; + (data, chosen) + }) + } else { + let raw = run_batch_blob_with_timeout( + "BatchReadBlobs", + digest_copy, + "Error reading from store", + store_ref.get_part_unchunked(digest_copy, 0, None), + ) + .await; + match raw { + Ok(raw_data) if client_accepts_zstd => { + let data_for_compression = raw_data.clone(); + match spawn_blocking!("cas_encode_compressed_download", move || { + crate::wire_compression::compress_for_batch_read( + data_for_compression, + ) + }) + .await + { + Ok(compressed_result) => Ok(compressed_result), + Err(e) => { + warn!("Wire compression task failed for digest {:?}, falling back to identity: {}", digest, e); + Ok((raw_data, compressor::Value::Identity)) + } } } - } else { - (raw_data, compressor::Value::Identity) - }; - - (GrpcStatus::default(), output_data, chosen_compressor) + Ok(raw_data) => Ok((raw_data, compressor::Value::Identity)), + Err(e) => Err(e), + } + }; + let (status, data, response_compressor) = match outcome { + Ok((data, chosen_compressor)) => { + (GrpcStatus::default(), data, chosen_compressor) } Err(mut e) => { if e.code == Code::NotFound { - // Trim the error code. Not Found is quite common and we don't want to send a large - // error (debug) message for something that is common. We resize to just the last - // message as it will be the most relevant. + // Trim the message chain. Not Found is quite common and + // we don't want to send a large error (debug) message + // for something that is common. We resize to just the + // last message as it will be the most relevant. e.messages.resize_with(1, String::new); } (e.into(), Bytes::new(), compressor::Value::Identity) diff --git a/nativelink-service/src/wire_compression.rs b/nativelink-service/src/wire_compression.rs index 1e6c5facd..a921c6513 100644 --- a/nativelink-service/src/wire_compression.rs +++ b/nativelink-service/src/wire_compression.rs @@ -16,36 +16,30 @@ //! //! This module handles compression/decompression of blob data on the gRPC wire //! between client and server, per the REAPI compressed-blobs specification. -//! This is orthogonal to at-rest compression (`CompressionStore` with LZ4). +//! +//! Wire compression is independent of at-rest compression, but not always +//! disjoint from it: a store whose physical representation already matches the +//! negotiated wire representation can serve and accept those bytes directly. +//! Such a store advertises itself through +//! [`WireCompressionStore`](nativelink_util::store_trait::WireCompressionStore), +//! and the `ByteStream`/CAS services use that capability to skip the codecs +//! here. See [`wire_compressor_capability`]. use std::collections::HashSet; -use std::io::Read; use bytes::Bytes; use nativelink_config::cas_server::{CapabilitiesConfig, InstanceName, WithInstanceName}; -use nativelink_error::{Code, Error, ResultExt, make_err, make_input_err}; +use nativelink_error::{Code, Error, make_err, make_input_err}; use nativelink_proto::build::bazel::remote::execution::v2::compressor; -use nativelink_util::buf_channel::{DropCloserReadHalf, DropCloserWriteHalf}; -use nativelink_util::common::DigestInfo; -use nativelink_util::digest_hasher::{DigestHasher, DigestHasherFunc}; use nativelink_util::spawn_blocking; +// The codecs are shared with client-side GrpcStore transfers; re-export so +// existing service callers keep their import paths. +pub use nativelink_util::wire_compression::{ + ZSTD_COMPRESSION_LEVEL, compress, decompress, stream_decode_compressed_upload, + stream_encode_compressed_download, wire_compressor_capability, +}; use tracing::warn; -/// Zstd compression level for wire compression. -/// Level 0 in the zstd crate means "use default" (currently 3). -/// We use an explicit level for clarity. -pub const ZSTD_COMPRESSION_LEVEL: i32 = 3; - -/// Upper bound on the buffer `decompress` reserves up front for a zstd blob. -/// `expected_size` comes from the client's claimed digest, so it must never be -/// used as an allocation hint directly: a small payload claiming a huge size -/// would otherwise force a large pre-emptive allocation before the real -/// decompressed length is ever known. We reserve `min(expected_size, this)` -/// so common payloads never reallocate while a hostile claim allocates at most -/// this. Sized comfortably above any honest `BatchUpdateBlobs` payload (the -/// only caller of this bulk path; large blobs stream through `ByteStream`). -const ZSTD_DECOMPRESS_PREALLOC_CAP: usize = 1024 * 1024; - /// Which instances accept and advertise REAPI compressed-blobs (zstd). /// /// Derived from `CapabilitiesConfig.remote_cache_compression` so that @@ -78,55 +72,6 @@ impl RemoteCacheCompressionInstances { } } -#[derive(Debug)] -pub struct BufChannelReader { - rx: DropCloserReadHalf, - chunk: Bytes, - chunk_offset: usize, -} - -impl BufChannelReader { - pub const fn new(rx: DropCloserReadHalf) -> Self { - Self { - rx, - chunk: Bytes::new(), - chunk_offset: 0, - } - } - - fn refill_chunk(&mut self) -> std::io::Result { - while self.chunk_offset == self.chunk.len() { - self.chunk = self.rx.blocking_recv().map_err(Error::to_std_err)?; - self.chunk_offset = 0; - if self.chunk.is_empty() { - return Ok(false); - } - } - - Ok(true) - } -} - -impl Read for BufChannelReader { - fn read(&mut self, output: &mut [u8]) -> std::io::Result { - if output.is_empty() { - return Ok(0); - } - - if !self.refill_chunk()? { - return Ok(0); - } - - let chunk_remaining = &self.chunk[self.chunk_offset..]; - let bytes_to_copy = output.len().min(chunk_remaining.len()); - - output[..bytes_to_copy].copy_from_slice(&chunk_remaining[..bytes_to_copy]); - self.chunk_offset += bytes_to_copy; - - Ok(bytes_to_copy) - } -} - /// Resolve a wire compressor from a URI compressor string (as it appears in /// the `compressed-blobs/{compressor}/...` resource name) and validate it /// against whether the instance supports remote cache compression. @@ -153,88 +98,6 @@ pub fn resolve_wire_compressor( } } -/// Compress data using the specified wire compressor. -/// -/// `data` is the raw (uncompressed) bytes from the store. -/// Returns the compressed bytes suitable for sending on the wire. -pub fn compress(data: Bytes, compressor_value: compressor::Value) -> Result { - match compressor_value { - compressor::Value::Identity => Ok(data), - compressor::Value::Zstd => { - let compressed = zstd::bulk::compress(&data, ZSTD_COMPRESSION_LEVEL) - .map_err(|e| make_err!(Code::Internal, "Zstd compression failed: {}", e))?; - Ok(Bytes::from(compressed)) - } - _ => Err(make_input_err!( - "Unsupported wire compressor for compression: {:?}", - compressor_value - )), - } -} - -/// Decompress data using the specified wire compressor. -/// -/// `data` is the compressed bytes received from the wire. -/// `expected_size` is the uncompressed size (from the client's digest). It is -/// the hard cap on the decompressed output, but never a direct allocation -/// hint: the buffer grows with the real decoded bytes so a small payload -/// claiming a huge size cannot force a large up-front allocation. -/// Returns the decompressed bytes suitable for storing. -pub fn decompress( - data: &[u8], - compressor_value: compressor::Value, - expected_size: usize, -) -> Result { - match compressor_value { - compressor::Value::Identity => { - if data.len() != expected_size { - return Err(make_err!( - Code::InvalidArgument, - "Identity data size {} does not match expected size {}", - data.len(), - expected_size - )); - } - Ok(Bytes::copy_from_slice(data)) - } - compressor::Value::Zstd => { - // Decode incrementally so `expected_size` (which is attacker - // controlled — it is the client's claimed digest size) can bound - // the output without being trusted as an allocation size. We - // reserve only `min(expected_size, ZSTD_DECOMPRESS_PREALLOC_CAP)`, - // then `take(expected_size + 1)` hard-caps the decoder so a - // decompression bomb is rejected as soon as it overshoots. This - // mirrors the real-byte-count validation the identity arm and the - // streaming upload path already perform. - let decoder = zstd::stream::read::Decoder::new(data) - .map_err(|e| make_err!(Code::InvalidArgument, "Zstd decompression failed: {e}"))?; - let mut output = Vec::with_capacity(expected_size.min(ZSTD_DECOMPRESS_PREALLOC_CAP)); - // `+ 1` lets an oversized stream produce one byte past the cap so - // the size check below rejects it rather than silently truncating. - let cap = u64::try_from(expected_size) - .err_tip(|| "expected_size did not fit in u64")? - .saturating_add(1); - decoder - .take(cap) - .read_to_end(&mut output) - .map_err(|e| make_err!(Code::InvalidArgument, "Zstd decompression failed: {e}"))?; - if output.len() != expected_size { - return Err(make_err!( - Code::InvalidArgument, - "Decompressed size {} does not match expected size {}", - output.len(), - expected_size - )); - } - Ok(Bytes::from(output)) - } - _ => Err(make_input_err!( - "Unsupported wire compressor for decompression: {:?}", - compressor_value - )), - } -} - #[must_use] pub fn compress_for_batch_read(data: Bytes) -> (Bytes, compressor::Value) { match compress(data.clone(), compressor::Value::Zstd) { @@ -290,106 +153,3 @@ pub async fn decompress_batch_update( other => Err(make_input_err!("Unsupported wire compressor: {:?}", other)), } } - -pub fn stream_decode_compressed_upload( - compressed_rx: DropCloserReadHalf, - wire_compressor: compressor::Value, - digest: DigestInfo, - digest_function: DigestHasherFunc, - mut tx: DropCloserWriteHalf, -) -> Result<(), Error> { - if wire_compressor != compressor::Value::Zstd { - return Err(make_input_err!( - "Streaming upload decompression only supports zstd, got {:?}", - wire_compressor - )); - } - - let expected_size = digest.size_bytes(); - let mut hasher = digest_function.hasher(); - let mut decoded_size = 0u64; - let mut buffer = vec![0u8; zstd::zstd_safe::DCtx::out_size()]; - - let reader = BufChannelReader::new(compressed_rx); - let mut decoder = zstd::stream::read::Decoder::new(reader) - .map_err(|e| make_err!(Code::InvalidArgument, "Zstd decompression failed: {}", e))?; - loop { - let read = decoder - .read(&mut buffer) - .map_err(|e| make_err!(Code::InvalidArgument, "Zstd decompression failed: {}", e))?; - if read == 0 { - break; - } - let read_u64 = - u64::try_from(read).err_tip(|| "Decoded chunk size was not convertible to u64")?; - decoded_size = decoded_size.checked_add(read_u64).ok_or_else(|| { - make_err!( - Code::InvalidArgument, - "Decoded compressed upload size overflow" - ) - })?; - if decoded_size > expected_size { - return Err(make_err!( - Code::InvalidArgument, - "Decoded compressed upload size {} bytes exceeds digest size {} bytes", - decoded_size, - expected_size - )); - } - hasher.update(&buffer[..read]); - tx.blocking_send(Bytes::copy_from_slice(&buffer[..read]))?; - } - - if decoded_size != expected_size { - return Err(make_err!( - Code::InvalidArgument, - "Decompressed size {} does not match expected size {}", - decoded_size, - expected_size - )); - } - let actual_digest = hasher.finalize_digest(); - if actual_digest != digest { - return Err(make_err!( - Code::InvalidArgument, - "Decompressed digest {} does not match expected digest {}", - actual_digest, - digest - )); - } - - tx.send_eof() - .err_tip(|| "Failed to send decompressed upload EOF")?; - Ok(()) -} - -pub fn stream_encode_compressed_download( - raw_rx: DropCloserReadHalf, - wire_compressor: compressor::Value, - mut tx: DropCloserWriteHalf, -) -> Result<(), Error> { - if wire_compressor != compressor::Value::Zstd { - return Err(make_input_err!( - "Streaming download compression only supports zstd, got {:?}", - wire_compressor - )); - } - - let reader = BufChannelReader::new(raw_rx); - let mut encoder = zstd::stream::read::Encoder::new(reader, ZSTD_COMPRESSION_LEVEL) - .map_err(|e| make_err!(Code::Internal, "Zstd compression failed: {}", e))?; - let mut buffer = vec![0u8; 64 * 1024]; - loop { - let read = encoder - .read(&mut buffer) - .map_err(|e| make_err!(Code::Internal, "Zstd compression failed: {}", e))?; - if read == 0 { - break; - } - tx.blocking_send(Bytes::copy_from_slice(&buffer[..read]))?; - } - - tx.send_eof() - .err_tip(|| "Failed to send compressed download EOF")?; - Ok(()) -} diff --git a/nativelink-service/tests/ac_server_test.rs b/nativelink-service/tests/ac_server_test.rs index 5c65b37f8..6d1434914 100644 --- a/nativelink-service/tests/ac_server_test.rs +++ b/nativelink-service/tests/ac_server_test.rs @@ -198,7 +198,9 @@ async fn one_item_update_test() -> Result<(), Box> { ..Default::default() }; - let size_bytes = get_encoded_proto_size(&action_result)? as i64; + let size_bytes = get_encoded_proto_size(&action_result)? + .try_into() + .unwrap_or(i64::MAX); let raw_response = update_action_result( &ac_server, diff --git a/nativelink-service/tests/bep_server_test.rs b/nativelink-service/tests/bep_server_test.rs index 9d0139408..8124130b5 100644 --- a/nativelink-service/tests/bep_server_test.rs +++ b/nativelink-service/tests/bep_server_test.rs @@ -48,7 +48,7 @@ use nativelink_util::common::encode_stream_proto; use nativelink_util::default_health_status_indicator; use nativelink_util::health_utils::HealthStatusIndicator; use nativelink_util::store_trait::{ - RemoveItemCallback, Store, StoreDriver, StoreKey, StoreLike, UploadSizeInfo, + RemoveCallback, Store, StoreDriver, StoreKey, StoreLike, UploadSizeInfo, }; use pretty_assertions::assert_eq; use prost::Message; @@ -154,10 +154,7 @@ impl StoreDriver for FlakyStore { self } - fn register_remove_callback( - self: Arc, - _callback: Arc, - ) -> Result<(), Error> { + fn register_remove_callback(self: Arc, _callback: RemoveCallback) -> Result<(), Error> { todo!(); } } @@ -433,7 +430,7 @@ async fn publish_build_tool_event_stream_test() -> Result<(), Box Result, Error> { Ok(store_manager) } +async fn make_verify_store_manager() -> Result, Error> { + let store_manager = Arc::new(StoreManager::new()); + store_manager.add_store( + "main_cas", + store_factory( + &StoreSpec::Verify(Box::new(VerifySpec { + verify_size: true, + verify_hash: true, + backend: StoreSpec::Memory(MemorySpec::default()), + })), + &store_manager, + None, + ) + .await?, + )?; + Ok(store_manager) +} + +/// Builds a [`StoreManager`] whose `main_cas` store is a pure-passthrough +/// zstd compression over an in-memory backend, so the `ByteStream` zstd fast +/// paths are exercised end-to-end. +async fn make_zstd_store_manager() -> Result, Error> { + let store_manager = Arc::new(StoreManager::new()); + let temp_path = make_temp_path("bytestream-zstd-store"); + std::fs::create_dir_all(&temp_path) + .map_err(|e| make_err!(Code::Internal, "Failed to create zstd temp dir: {e}"))?; + let spec = StoreSpec::Compression(Box::new(CompressionSpec { + backend: StoreSpec::Memory(MemorySpec::default()), + compression_algorithm: CompressionAlgorithm::Zstd(ZstdConfig { + temp_path, + max_compressed_upload_size: 512 * 1024 * 1024, + ..ZstdConfig::default() + }), + })); + store_manager.add_store( + "main_cas", + store_factory(&spec, &store_manager, None).await?, + )?; + Ok(store_manager) +} + +/// Gets the configured store's immediate wire-compression capability for +/// direct fast-path setup without coupling this service test to a store type. +fn wire_compression_store_of(store_manager: &StoreManager) -> Arc { + store_manager + .get_store("main_cas") + .expect("main_cas store missing") + .wire_compression_store() + .expect("main_cas store was not wire-compression capable") +} + fn make_bytestream_server( store_manager: &StoreManager, config: Option>>, @@ -90,6 +144,7 @@ fn make_bytestream_server_with_remote_cache_compression( config: ByteStreamConfig { cas_store: "main_cas".to_string(), persist_stream_on_disconnect_timeout_s: 0, + compressed_upload_idle_timeout_s: 0, max_bytes_per_stream: 1024, }, }] @@ -150,7 +205,7 @@ fn make_compressed_resource_name_with_compressor( hash: &str, data_len: impl core::fmt::Display, ) -> String { - format!("{INSTANCE_NAME}/uploads/{uuid}/compressed-blobs/{compressor}/{hash}/{data_len}") + format!("{INSTANCE_NAME}/uploads/{uuid}/compressed-blobs/{compressor}/sha256/{hash}/{data_len}") } fn sha256_hex(data: &[u8]) -> String { @@ -159,6 +214,12 @@ fn sha256_hex(data: &[u8]) -> String { hex::encode(hasher.finalize()) } +fn blake3_hex(data: &[u8]) -> String { + let mut hasher = DigestHasherFunc::Blake3.hasher(); + hasher.update(data); + hasher.finalize_digest().packed_hash().to_string() +} + async fn read_all_bytes( bs_server: &ByteStreamServer, read_request: ReadRequest, @@ -286,13 +347,13 @@ pub async fn chunked_stream_receives_all_data() -> Result<(), Box Result<(), Box Result<(), Box> { + const WRITE_DATA: &[u8] = b"sha256 cache entry without a digest segment"; + + let hash = sha256_hex(WRITE_DATA); + let store_manager = make_verify_store_manager().await?; + let bs_server = Arc::new(make_bytestream_server(&store_manager, None)?); + let write_resource_name = format!( + "{INSTANCE_NAME}/uploads/2dccbdca-8d6f-490c-8748-b3c723cf67a8/blobs/{hash}/{}", + WRITE_DATA.len() + ); + + let (tx, join_handle) = make_stream_and_writer_spawn(bs_server.clone(), None); + tx.send(Frame::data(encode_stream_proto(&WriteRequest { + resource_name: write_resource_name, + write_offset: 0, + finish_write: true, + data: WRITE_DATA.to_vec().into(), + })?)) + .await?; + drop(tx); + join_handle.await??; + + let cached_data = read_all_bytes( + &bs_server, + ReadRequest { + resource_name: format!("{INSTANCE_NAME}/blobs/{hash}/{}", WRITE_DATA.len()), + read_offset: 0, + read_limit: 0, + }, + ) + .await?; + assert_eq!(cached_data, WRITE_DATA); + Ok(()) +} + +#[nativelink_test] +pub async fn explicit_blake3_write_overrides_sha256_default_with_warning() +-> Result<(), Box> { + const WRITE_DATA: &[u8] = b"content hashed with blake3"; + + let hash = blake3_hex(WRITE_DATA); + let digest = DigestInfo::try_new(&hash, WRITE_DATA.len())?; + let store_manager = make_verify_store_manager().await?; + let bs_server = Arc::new(make_bytestream_server(&store_manager, None)?); + let store = store_manager.get_store("main_cas").unwrap(); + let resource_name = format!( + "{INSTANCE_NAME}/uploads/9eb44e87-1320-40e7-9154-bf19f3fdf94a/blobs/blake3/{hash}/{}", + WRITE_DATA.len() + ); + + let (tx, join_handle) = make_stream_and_writer_spawn(bs_server, None); + tx.send(Frame::data(encode_stream_proto(&WriteRequest { + resource_name, + write_offset: 0, + finish_write: true, + data: WRITE_DATA.to_vec().into(), + })?)) + .await?; + drop(tx); + + join_handle.await??; + assert_eq!(store.get_part_unchunked(digest, 0, None).await?, WRITE_DATA); + assert!(logs_contain( + "clients using different digest functions generate different cache keys and will not share cache hits" + )); + Ok(()) +} + +#[nativelink_test] +pub async fn resumed_blake3_write_keeps_declared_digest_function() +-> Result<(), Box> { + const WRITE_DATA: &[u8] = b"resumed content hashed with blake3"; + const BYTE_SPLIT_OFFSET: usize = 12; + + let hash = blake3_hex(WRITE_DATA); + let digest = DigestInfo::try_new(&hash, WRITE_DATA.len())?; + let store_manager = make_verify_store_manager().await?; + let bs_server = Arc::new(make_bytestream_server(&store_manager, None)?); + let store = store_manager.get_store("main_cas").unwrap(); + let resource_name = format!( + "{INSTANCE_NAME}/uploads/4dcec57e-1389-4ab5-b188-4a59f22ceb4b/blobs/blake3/{hash}/{}", + WRITE_DATA.len() + ); + + let (tx, join_handle) = make_stream_and_writer_spawn(bs_server.clone(), None); + tx.send(Frame::data(encode_stream_proto(&WriteRequest { + resource_name: resource_name.clone(), + write_offset: 0, + finish_write: false, + data: WRITE_DATA[..BYTE_SPLIT_OFFSET].to_vec().into(), + })?)) + .await?; + drop(tx); + assert!( + join_handle.await?.is_err(), + "the disconnected partial write should remain resumable" + ); + + let (tx, join_handle) = make_stream_and_writer_spawn(bs_server, None); + tx.send(Frame::data(encode_stream_proto(&WriteRequest { + resource_name, + write_offset: BYTE_SPLIT_OFFSET.try_into()?, + finish_write: true, + data: WRITE_DATA[BYTE_SPLIT_OFFSET..].to_vec().into(), + })?)) + .await?; + drop(tx); + + join_handle.await??; + assert_eq!(store.get_part_unchunked(digest, 0, None).await?, WRITE_DATA); + Ok(()) +} + #[nativelink_test] pub async fn resume_write_success() -> Result<(), Box> { const WRITE_DATA: &str = "12456789abcdefghijk"; @@ -380,7 +556,7 @@ pub async fn resume_write_success() -> Result<(), Box> { make_stream_and_writer_spawn(bs_server, Some(CompressionEncoding::Gzip)); { // Write the remainder of our data. - write_request.write_offset = BYTE_SPLIT_OFFSET as i64; + write_request.write_offset = BYTE_SPLIT_OFFSET.try_into().unwrap_or(i64::MAX); write_request.finish_write = true; write_request.data = WRITE_DATA[BYTE_SPLIT_OFFSET..].into(); tx.send(Frame::data(encode_stream_proto(&write_request)?)) @@ -461,7 +637,7 @@ pub async fn restart_write_success() -> Result<(), Box> } { // Write the remainder of our data. - write_request.write_offset = BYTE_SPLIT_OFFSET as i64; + write_request.write_offset = BYTE_SPLIT_OFFSET.try_into().unwrap_or(i64::MAX); write_request.finish_write = true; write_request.data = WRITE_DATA[BYTE_SPLIT_OFFSET..].into(); tx.send(Frame::data(encode_stream_proto(&write_request)?)) @@ -540,7 +716,7 @@ pub async fn restart_mid_stream_write_success() -> Result<(), Box Result<(), Box } { // Write data it already has. - write_request.write_offset = (BYTE_SPLIT_OFFSET - 1) as i64; + write_request.write_offset = (BYTE_SPLIT_OFFSET - 1).try_into().unwrap_or(i64::MAX); write_request.data = WRITE_DATA[(BYTE_SPLIT_OFFSET - 1)..].into(); tx.send(Frame::data(encode_stream_proto(&write_request)?)) .await?; @@ -1103,7 +1280,7 @@ pub async fn out_of_order_data_fails() -> Result<(), Box ); { // Make sure stream was closed. - write_request.write_offset = (BYTE_SPLIT_OFFSET - 1) as i64; + write_request.write_offset = (BYTE_SPLIT_OFFSET - 1).try_into().unwrap_or(i64::MAX); write_request.data = WRITE_DATA[(BYTE_SPLIT_OFFSET - 1)..].into(); assert!( tx.send(Frame::data(encode_stream_proto(&write_request)?)) @@ -1226,7 +1403,7 @@ pub async fn chunked_stream_reads_small_set_of_data() -> Result<(), Box Result<(), Box Result<(), Box Result<(), Box Result<(), Box Result<(), Box Result<(), Box> +{ + let raw_data = Bytes::from("byte-for-byte zstd passthrough ".repeat(256)); + let hash = sha256_hex(raw_data.as_ref()); + // A known zstd stream. With compression_level=None + max_recompression_size=0 + // the store is a pure passthrough, so these exact bytes are what is stored. + let compressed = Bytes::from(zstd::bulk::compress(raw_data.as_ref(), 3)?); + assert_ne!( + compressed.as_ref(), + raw_data.as_ref(), + "test data must actually compress" + ); + + let store_manager = make_zstd_store_manager().await?; + let bs_server = Arc::new( + make_bytestream_server_with_remote_cache_compression(store_manager.as_ref(), None, true) + .expect("Failed to make server"), + ); + let wire_store = wire_compression_store_of(store_manager.as_ref()); + + let digest = DigestInfo::try_new(&hash, raw_data.len())?; + wire_store + .update_compressed_oneshot( + digest, + DigestHasherFunc::Sha256, + WireCompressor::Zstd, + compressed.clone(), + ) + .await?; + + let read_data = read_all_bytes( + bs_server.as_ref(), + ReadRequest { + resource_name: format!( + "{}/compressed-blobs/zstd/{}/{}", + INSTANCE_NAME, + hash, + raw_data.len() + ), + read_offset: 0, + read_limit: 0, + }, + ) + .await?; + + assert_eq!( + read_data.as_slice(), + compressed.as_ref(), + "offset-0 compressed read from a ZstdStore must be byte-for-byte identical to what was stored" + ); + let decoded = zstd::bulk::decompress(&read_data, raw_data.len())?; + assert_eq!( + decoded.as_slice(), + raw_data.as_ref(), + "returned zstd stream must decode to the original blob" + ); + + Ok(()) +} + +/// A compressed upload to a `ZstdStore` instance stores the client's zstd bytes +/// byte-for-byte, reports the compressed wire byte count as `committed_size`, and +/// a re-upload of an already-present blob still completes. +#[nativelink_test] +pub async fn zstd_store_compressed_write_round_trip() -> Result<(), Box> { + let raw_data = "zstd store compressed write round trip ".repeat(256); + let hash = sha256_hex(raw_data.as_bytes()); + let compressed = zstd::bulk::compress(raw_data.as_bytes(), 3)?; + assert!(compressed.len() < raw_data.len(), "test data must compress"); + + let store_manager = make_zstd_store_manager().await?; + let bs_server = Arc::new( + make_bytestream_server_with_remote_cache_compression(store_manager.as_ref(), None, true) + .expect("Failed to make server"), + ); + + // Upload the compressed blob. + let (tx, join_handle) = make_stream_and_writer_spawn(bs_server.clone(), None); + tx.send(Frame::data(encode_stream_proto(&WriteRequest { + resource_name: make_compressed_resource_name( + "4dcec57e-1389-4ab5-b188-4a59f22ceb60", + &hash, + raw_data.len(), + ), + write_offset: 0, + finish_write: true, + data: compressed.clone().into(), + })?)) + .await?; + let server_result = join_handle + .await + .expect("Failed to join") + .expect("Failed write"); + assert_eq!( + server_result.into_inner().committed_size, + i64::try_from(compressed.len()).unwrap(), + "compressed write to a ZstdStore must report the compressed wire byte count" + ); + + // A subsequent compressed read returns the exact stored zstd bytes. + let read_data = read_all_bytes( + bs_server.as_ref(), + ReadRequest { + resource_name: format!( + "{}/compressed-blobs/zstd/{}/{}", + INSTANCE_NAME, + hash, + raw_data.len() + ), + read_offset: 0, + read_limit: 0, + }, + ) + .await?; + assert_eq!( + read_data.as_slice(), + compressed.as_slice(), + "read-back must be byte-for-byte identical to the uploaded zstd stream" + ); + assert_eq!( + zstd::bulk::decompress(&read_data, raw_data.len())?.as_slice(), + raw_data.as_bytes() + ); + + // Re-upload an already-present blob; it must still complete (REAPI allows + // -1 or the byte count for an already-present blob). + let (tx2, join2) = make_stream_and_writer_spawn(bs_server.clone(), None); + tx2.send(Frame::data(encode_stream_proto(&WriteRequest { + resource_name: make_compressed_resource_name( + "4dcec57e-1389-4ab5-b188-4a59f22ceb61", + &hash, + raw_data.len(), + ), + write_offset: 0, + finish_write: true, + data: compressed.clone().into(), + })?)) + .await?; + let committed_size = join2 + .await + .expect("Failed to join") + .expect("Failed re-upload write") + .into_inner() + .committed_size; + assert!( + committed_size == -1 || committed_size == i64::try_from(compressed.len()).unwrap(), + "re-upload committed_size must be -1 or the compressed byte count {}; got {committed_size}", + compressed.len() + ); + + Ok(()) +} + +/// A compressed read at offset > 0 from a `ZstdStore` instance takes the +/// fallback path (`StoreDriver::get_part` decompresses, service re-encodes) and +/// returns a valid zstd stream decoding to the uncompressed suffix. +#[nativelink_test] +pub async fn zstd_store_compressed_read_offset_uses_fallback() +-> Result<(), Box> { + let raw_data = Bytes::from( + (0usize..4096) + .map(|i| u8::try_from((i * 31 + i / 7) % 251).expect("modulo 251 fits in u8")) + .collect::>(), + ); + let hash = sha256_hex(raw_data.as_ref()); + let compressed = Bytes::from(zstd::bulk::compress(raw_data.as_ref(), 3)?); + + let store_manager = make_zstd_store_manager().await?; + let bs_server = Arc::new( + make_bytestream_server_with_remote_cache_compression(store_manager.as_ref(), None, true) + .expect("Failed to make server"), + ); + let wire_store = wire_compression_store_of(store_manager.as_ref()); + + let digest = DigestInfo::try_new(&hash, raw_data.len())?; + wire_store + .update_compressed_oneshot( + digest, + DigestHasherFunc::Sha256, + WireCompressor::Zstd, + compressed, + ) + .await?; + + let read_offset = 100usize; + let ranged_data = read_all_bytes( + bs_server.as_ref(), + ReadRequest { + resource_name: format!( + "{}/compressed-blobs/zstd/{}/{}", + INSTANCE_NAME, + hash, + raw_data.len() + ), + read_offset: i64::try_from(read_offset).unwrap(), + read_limit: 0, + }, + ) + .await?; + + let decoded = zstd::bulk::decompress(&ranged_data, raw_data.len() - read_offset)?; + assert_eq!( + decoded.as_slice(), + &raw_data.as_ref()[read_offset..], + "offset > 0 compressed read must decode to the uncompressed suffix" + ); + + Ok(()) +} + +/// An identity (non-compressed) write + read round trip through a `ZstdStore` +/// instance returns the correct raw bytes via the `StoreDriver` path. +#[nativelink_test] +pub async fn zstd_store_identity_write_and_read_round_trip() +-> Result<(), Box> { + let raw_data = Bytes::from("identity round trip through a zstd store ".repeat(64)); + let hash = sha256_hex(raw_data.as_ref()); + + let store_manager = make_zstd_store_manager().await?; + let bs_server = Arc::new( + make_bytestream_server(store_manager.as_ref(), None).expect("Failed to make server"), + ); + + let (tx, join_handle) = make_stream_and_writer_spawn(bs_server.clone(), None); + tx.send(Frame::data(encode_stream_proto(&WriteRequest { + resource_name: format!( + "{}/uploads/{}/blobs/{}/{}", + INSTANCE_NAME, + "4dcec57e-1389-4ab5-b188-4a59f22ceb62", + hash, + raw_data.len() + ), + write_offset: 0, + finish_write: true, + data: raw_data.clone(), + })?)) + .await?; + let server_result = join_handle + .await + .expect("Failed to join") + .expect("Failed write"); + assert_eq!( + server_result.into_inner().committed_size, + i64::try_from(raw_data.len()).unwrap() + ); + + let read_data = read_all_bytes( + bs_server.as_ref(), + ReadRequest { + resource_name: format!("{}/blobs/{}/{}", INSTANCE_NAME, hash, raw_data.len()), + read_offset: 0, + read_limit: i64::try_from(raw_data.len()).unwrap(), + }, + ) + .await?; + assert_eq!( + read_data.as_slice(), + raw_data.as_ref(), + "identity round trip through a ZstdStore must return the raw bytes" + ); + + Ok(()) +} + +/// A compressed read with a non-zero `read_limit` against a `ZstdStore` instance +/// must be rejected with `INVALID_ARGUMENT` (regression: the guard runs before the +/// fast path). +#[nativelink_test] +pub async fn zstd_store_compressed_read_rejects_nonzero_read_limit() +-> Result<(), Box> { + let raw_data = Bytes::from("read_limit rejection over a zstd store ".repeat(64)); + let hash = sha256_hex(raw_data.as_ref()); + let compressed = Bytes::from(zstd::bulk::compress(raw_data.as_ref(), 3)?); + + let store_manager = make_zstd_store_manager().await?; + let bs_server = + make_bytestream_server_with_remote_cache_compression(store_manager.as_ref(), None, true) + .expect("Failed to make server"); + let wire_store = wire_compression_store_of(store_manager.as_ref()); + + let digest = DigestInfo::try_new(&hash, raw_data.len())?; + wire_store + .update_compressed_oneshot( + digest, + DigestHasherFunc::Sha256, + WireCompressor::Zstd, + compressed, + ) + .await?; + + let Err(status) = bs_server + .read(Request::new(ReadRequest { + resource_name: format!( + "{}/compressed-blobs/zstd/{}/{}", + INSTANCE_NAME, + hash, + raw_data.len() + ), + read_offset: 0, + read_limit: 1, + })) + .await + else { + panic!("compressed read with read_limit should fail"); + }; + + assert_eq!(status.code(), Code::InvalidArgument); + assert!( + status.message().contains("read_limit must be 0"), + "unexpected error: {}", + status.message() + ); + + Ok(()) +} + +/// Builds a [`StoreManager`] whose `main_cas` store is a pure-passthrough +/// `ZstdStore` over memory with a configurable staged-upload bound. +async fn make_zstd_store_manager_with_staging( + max_concurrent_staged_uploads: usize, +) -> Result, Error> { + let store_manager = Arc::new(StoreManager::new()); + let temp_path = make_temp_path("bytestream-zstd-idle"); + std::fs::create_dir_all(&temp_path) + .map_err(|e| make_err!(Code::Internal, "Failed to create zstd temp dir: {e}"))?; + let spec = StoreSpec::Compression(Box::new(CompressionSpec { + backend: StoreSpec::Memory(MemorySpec::default()), + compression_algorithm: CompressionAlgorithm::Zstd(ZstdConfig { + temp_path, + max_compressed_upload_size: 512 * 1024 * 1024, + max_concurrent_staged_uploads, + ..ZstdConfig::default() + }), + })); + store_manager.add_store( + "main_cas", + store_factory(&spec, &store_manager, None).await?, + )?; + Ok(store_manager) +} + +/// A compressed `ByteStream` client that acquires the single staging slot and then +/// stalls (never sends `finish_write`, never disconnects) is timed out with +/// `DeadlineExceeded`, its staging permit is released, and a subsequent valid +/// upload behind the `max_concurrent_staged_uploads = 1` bound completes. +/// +/// Uses a short real-time (1s) idle timeout rather than a paused clock: the +/// stalled upload parks a `spawn_blocking` recv thread inside the store, and +/// tokio's `start_paused` auto-advance will not fire the idle timer while a +/// blocking task is outstanding — so a paused clock would deadlock here. +#[nativelink_test] +async fn zstd_compressed_upload_idle_timeout_frees_staging_slot() +-> Result<(), Box> { + let store_manager = make_zstd_store_manager_with_staging(1).await?; + // Short idle timeout; under start_paused the virtual clock advances past it. + let config = vec![WithInstanceName { + instance_name: INSTANCE_NAME.to_string(), + config: ByteStreamConfig { + cas_store: "main_cas".to_string(), + persist_stream_on_disconnect_timeout_s: 1, + // The knob under test: how long to wait for the next WriteRequest + // of a compressed upload. + compressed_upload_idle_timeout_s: 1, + max_bytes_per_stream: 1024, + }, + }]; + let bs_server = Arc::new(make_bytestream_server_with_remote_cache_compression( + store_manager.as_ref(), + Some(config), + true, + )?); + + let raw = "idle timeout regression payload ".repeat(64); + let compressed = zstd::bulk::compress(raw.as_bytes(), 3)?; + let hash = sha256_hex(raw.as_bytes()); + + // --- Upload A: send a partial chunk, then stall (keep tx alive). --- + let resource_a = + make_compressed_resource_name("aaaaaaaa-1111-2222-3333-444444444444", &hash, raw.len()); + let (tx_a, join_a) = make_stream_and_writer_spawn(bs_server.clone(), None); + tx_a.send(Frame::data(encode_stream_proto(&WriteRequest { + resource_name: resource_a, + write_offset: 0, + finish_write: false, + data: Bytes::copy_from_slice(&compressed[..compressed.len() / 2]), + })?)) + .await?; + + // Do not send more and do not drop tx_a: the server is left waiting for the + // next WriteRequest. The idle timeout must fire (DeadlineExceeded), not an + // InvalidArgument "stream ended without finish_write". + let status_a = join_a + .await + .expect("join A") + .expect_err("stalled upload A must fail"); + assert_eq!( + status_a.code(), + Code::DeadlineExceeded, + "an idle-stalled compressed upload must time out with DeadlineExceeded, got: {status_a:?}" + ); + drop(tx_a); + + // --- Upload B: a full valid upload must complete (slot was released). --- + let resource_b = + make_compressed_resource_name("bbbbbbbb-1111-2222-3333-444444444444", &hash, raw.len()); + let (tx_b, join_b) = make_stream_and_writer_spawn(bs_server.clone(), None); + tx_b.send(Frame::data(encode_stream_proto(&WriteRequest { + resource_name: resource_b, + write_offset: 0, + finish_write: true, + data: compressed.clone().into(), + })?)) + .await?; + join_b + .await + .expect("join B") + .expect("valid upload B behind the freed slot must succeed"); + + let store = store_manager.get_store("main_cas").unwrap(); + let digest = DigestInfo::try_new(&hash, raw.len())?; + assert_eq!( + store.get_part_unchunked(digest, 0, None).await?.as_ref(), + raw.as_bytes(), + "the recovered upload must store the decompressed bytes" + ); + Ok(()) +} + +/// A client that keeps making progress is NOT idle-timed-out even when the whole +/// upload takes longer than the idle timeout: the deadline is per-WriteRequest +/// and resets on every chunk. Real-time test with sub-timeout inter-chunk gaps +/// whose sum exceeds the timeout. +#[nativelink_test] +async fn zstd_compressed_upload_making_progress_is_not_idle_timed_out() +-> Result<(), Box> { + let store_manager = make_zstd_store_manager_with_staging(1).await?; + let config = vec![WithInstanceName { + instance_name: INSTANCE_NAME.to_string(), + config: ByteStreamConfig { + cas_store: "main_cas".to_string(), + persist_stream_on_disconnect_timeout_s: 1, + // The knob under test: how long to wait for the next WriteRequest + // of a compressed upload. + compressed_upload_idle_timeout_s: 1, + max_bytes_per_stream: 1024, + }, + }]; + let bs_server = Arc::new(make_bytestream_server_with_remote_cache_compression( + store_manager.as_ref(), + Some(config), + true, + )?); + + let raw = "progress keeps the stream alive ".repeat(256); + let compressed = zstd::bulk::compress(raw.as_bytes(), 3)?; + let hash = sha256_hex(raw.as_bytes()); + let resource_name = + make_compressed_resource_name("cccccccc-1111-2222-3333-444444444444", &hash, raw.len()); + let (tx, join_handle) = make_stream_and_writer_spawn(bs_server.clone(), None); + + // Five chunks, ~300ms apart: each inter-chunk gap is well under the 1s idle + // timeout, but their sum (~1.2s) exceeds it. A per-message deadline that + // resets on progress lets this succeed; a whole-upload deadline would not. + let chunk_count = 5usize; + let chunk_len = compressed.len().div_ceil(chunk_count); + let mut offset = 0usize; + while offset < compressed.len() { + let end = (offset + chunk_len).min(compressed.len()); + tx.send(Frame::data(encode_stream_proto(&WriteRequest { + resource_name: resource_name.clone(), + write_offset: i64::try_from(offset).unwrap(), + finish_write: end == compressed.len(), + data: Bytes::copy_from_slice(&compressed[offset..end]), + })?)) + .await?; + offset = end; + if offset < compressed.len() { + tokio::time::sleep(core::time::Duration::from_millis(300)).await; + } + } + + join_handle + .await + .expect("join") + .expect("a continuously-progressing upload must not be idle-timed-out"); + + let store = store_manager.get_store("main_cas").unwrap(); + let digest = DigestInfo::try_new(&hash, raw.len())?; + assert_eq!( + store.get_part_unchunked(digest, 0, None).await?.as_ref(), + raw.as_bytes(), + "the progressive upload must store the decompressed bytes" + ); + Ok(()) +} diff --git a/nativelink-service/tests/capabilities_server_test.rs b/nativelink-service/tests/capabilities_server_test.rs index 25e8e4313..0b5aaf932 100644 --- a/nativelink-service/tests/capabilities_server_test.rs +++ b/nativelink-service/tests/capabilities_server_test.rs @@ -22,7 +22,7 @@ use nativelink_config::cas_server::{ use nativelink_macro::nativelink_test; use nativelink_proto::build::bazel::remote::execution::v2::capabilities_server::Capabilities; use nativelink_proto::build::bazel::remote::execution::v2::{ - GetCapabilitiesRequest, ServerCapabilities, compressor, + GetCapabilitiesRequest, ServerCapabilities, compressor, digest_function, }; use nativelink_scheduler::known_platform_property_provider::KnownPlatformPropertyProvider; use nativelink_scheduler::mock_scheduler::MockActionScheduler; @@ -82,6 +82,14 @@ async fn compression_only_instance_advertises_zstd_cache_capabilities() .cache_capabilities .expect("cache capabilities should be set"); + assert_eq!( + cache_capabilities.digest_functions, + vec![ + digest_function::Value::Sha256 as i32, + digest_function::Value::Blake3 as i32, + ], + "cache clients must be told that both digest functions are supported" + ); assert_eq!( cache_capabilities.supported_compressors, vec![compressor::Value::Zstd as i32] @@ -146,6 +154,19 @@ async fn remote_execution_instance_advertises_execution_capabilities_and_node_pr .expect("execution capabilities should be set"); assert!(execution_capabilities.exec_enabled); + assert_eq!( + execution_capabilities.digest_function, + digest_function::Value::Sha256 as i32, + "the legacy execution digest field should report the server default" + ); + assert_eq!( + execution_capabilities.digest_functions, + vec![ + digest_function::Value::Sha256 as i32, + digest_function::Value::Blake3 as i32, + ], + "execution clients must be told that both digest functions are supported" + ); assert_eq!( execution_capabilities.supported_node_properties, expected_properties diff --git a/nativelink-service/tests/cas_server_test.rs b/nativelink-service/tests/cas_server_test.rs index f8352ad2b..b556e8d54 100644 --- a/nativelink-service/tests/cas_server_test.rs +++ b/nativelink-service/tests/cas_server_test.rs @@ -21,7 +21,7 @@ use async_trait::async_trait; use bytes::Bytes; use futures::StreamExt; use nativelink_config::cas_server::WithInstanceName; -use nativelink_config::stores::{MemorySpec, StoreSpec}; +use nativelink_config::stores::{MemorySpec, StoreSpec, ZstdConfig}; use nativelink_error::Error; use nativelink_macro::nativelink_test; use nativelink_metric::MetricsComponent; @@ -38,13 +38,15 @@ use nativelink_service::cas_server::CasServer; use nativelink_service::wire_compression::RemoteCacheCompressionInstances; use nativelink_store::ac_utils::serialize_and_upload_message; use nativelink_store::default_store_factory::store_factory; +use nativelink_store::memory_store::MemoryStore; use nativelink_store::store_manager::StoreManager; +use nativelink_store::zstd_store::ZstdStore; use nativelink_util::buf_channel::{DropCloserReadHalf, DropCloserWriteHalf}; -use nativelink_util::common::DigestInfo; +use nativelink_util::common::{DigestInfo, make_temp_path}; use nativelink_util::digest_hasher::{DigestHasher, DigestHasherFunc}; use nativelink_util::health_utils::{HealthStatusIndicator, default_health_status_indicator}; use nativelink_util::store_trait::{ - RemoveItemCallback, Store, StoreDriver, StoreKey, StoreLike, UploadSizeInfo, + RemoveCallback, Store, StoreDriver, StoreKey, StoreLike, UploadSizeInfo, }; use pretty_assertions::assert_eq; use prost::Message; @@ -138,7 +140,7 @@ async fn store_one_item_existence() -> Result<(), Box> { instance_name: INSTANCE_NAME.to_string(), blob_digests: vec![Digest { hash: HASH1.to_string(), - size_bytes: VALUE.len() as i64, + size_bytes: VALUE.len().try_into().unwrap_or(i64::MAX), }], digest_function: digest_function::Value::Sha256.into(), })) @@ -166,15 +168,15 @@ async fn has_three_requests_one_bad_hash() -> Result<(), Box Result<(), Box> { let digest = Digest { hash: HASH1.to_string(), - size_bytes: VALUE2.len() as i64, + size_bytes: VALUE2.len().try_into().unwrap_or(i64::MAX), }; store @@ -256,11 +258,11 @@ async fn batch_read_blobs_read_two_blobs_success_one_fail() let digest1 = Digest { hash: HASH1.to_string(), - size_bytes: VALUE1.len() as i64, + size_bytes: VALUE1.len().try_into().unwrap_or(i64::MAX), }; let digest2 = Digest { hash: HASH2.to_string(), - size_bytes: VALUE2.len() as i64, + size_bytes: VALUE2.len().try_into().unwrap_or(i64::MAX), }; { // Insert dummy data. @@ -614,11 +616,11 @@ async fn batch_update_blobs_two_items_existence_with_third_missing() let digest1 = Digest { hash: HASH1.to_string(), - size_bytes: VALUE1.len() as i64, + size_bytes: VALUE1.len().try_into().unwrap_or(i64::MAX), }; let digest2 = Digest { hash: HASH2.to_string(), - size_bytes: VALUE2.len() as i64, + size_bytes: VALUE2.len().try_into().unwrap_or(i64::MAX), }; { @@ -679,12 +681,12 @@ async fn batch_update_blobs_two_items_existence_with_third_missing() blob_digests: vec![ Digest { hash: HASH1.to_string(), - size_bytes: VALUE1.len() as i64, + size_bytes: VALUE1.len().try_into().unwrap_or(i64::MAX), }, missing_digest.clone(), Digest { hash: HASH2.to_string(), - size_bytes: VALUE2.len() as i64, + size_bytes: VALUE2.len().try_into().unwrap_or(i64::MAX), }, ], digest_function: digest_function::Value::Sha256.into(), @@ -752,10 +754,7 @@ impl StoreDriver for StallStore { self } - fn register_remove_callback( - self: Arc, - _callback: Arc, - ) -> Result<(), Error> { + fn register_remove_callback(self: Arc, _callback: RemoveCallback) -> Result<(), Error> { Ok(()) } } @@ -789,7 +788,7 @@ async fn batch_update_blobs_per_blob_timeout_returns_deadline_exceeded() let digest = Digest { hash: HASH1.to_string(), - size_bytes: VALUE.len() as i64, + size_bytes: VALUE.len().try_into().unwrap_or(i64::MAX), }; let raw_response = cas_server .batch_update_blobs(Request::new(BatchUpdateBlobsRequest { @@ -921,7 +920,7 @@ async fn batch_update_blobs_zstd_rejected_when_remote_cache_compression_disabled let compressed_data = zstd::bulk::compress(raw_data, 3)?; let digest = Digest { hash: HASH1.to_string(), - size_bytes: raw_data.len() as i64, + size_bytes: raw_data.len().try_into().unwrap_or(i64::MAX), }; let Err(status) = cas_server @@ -958,7 +957,7 @@ async fn batch_read_blobs_zstd_compressed() -> Result<(), Box = "hello world ".repeat(100).into_bytes(); - let raw_size = raw_data.len() as i64; + let raw_size = raw_data.len().try_into().unwrap_or(i64::MAX); // Compute the sha256 digest. let mut hasher = DigestHasherFunc::Sha256.hasher(); @@ -1012,7 +1011,7 @@ async fn batch_read_blobs_zstd_falls_back_to_identity_when_not_smaller() let digest = Digest { hash: HASH1.to_string(), - size_bytes: raw_data.len() as i64, + size_bytes: raw_data.len().try_into().unwrap_or(i64::MAX), }; let response = cas_server .batch_read_blobs(Request::new(BatchReadBlobsRequest { @@ -1034,6 +1033,387 @@ async fn batch_read_blobs_zstd_falls_back_to_identity_when_not_smaller() Ok(()) } +/// A passthrough zstd configuration over an in-memory backend with no +/// recompression. +fn zstd_instance_spec(temp_path: String) -> ZstdConfig { + ZstdConfig { + temp_path, + max_compressed_upload_size: 512 * 1024 * 1024, + ..ZstdConfig::default() + } +} + +/// Builds a zstd-compressing `CasServer` over memory with wire compression +/// enabled, returning the concrete store so tests can seed it directly. +fn make_zstd_instance_cas_server() -> Result<(CasServer, Arc), Error> { + let temp_path = make_temp_path("cas_server_zstd_instance"); + std::fs::create_dir_all(&temp_path).expect("create temp dir"); + let inner = Store::new(MemoryStore::new(&MemorySpec::default())); + let zstd_store = ZstdStore::new(&zstd_instance_spec(temp_path), inner)?; + let store_manager = Arc::new(StoreManager::new()); + store_manager.add_store("main_cas", Store::new(zstd_store.clone()))?; + let cas_server = make_cas_server_with_zstd(&store_manager)?; + Ok((cas_server, zstd_store)) +} + +fn sha256_digest(data: &[u8]) -> (DigestInfo, Digest) { + let mut hasher = DigestHasherFunc::Sha256.hasher(); + DigestHasher::update(&mut hasher, data); + let digest_info = hasher.finalize_digest(); + let digest = Digest::from(&digest_info); + (digest_info, digest) +} + +/// High-entropy (incompressible) bytes generated by a deterministic xorshift. +fn incompressible_bytes(len: usize) -> Vec { + let mut state: u64 = 0x2545_F491_4F6C_DD1D; + (0..len) + .map(|_| { + state ^= state << 13; + state ^= state >> 7; + state ^= state << 17; + state.to_le_bytes()[0] + }) + .collect() +} + +#[nativelink_test] +async fn batch_read_zstd_instance_passthrough_and_identity() +-> Result<(), Box> { + let (cas_server, zstd_store) = make_zstd_instance_cas_server()?; + + // Compressible blob: seed the store with the client's zstd bytes verbatim + // (level 19, which the passthrough store keeps as-is). + let raw: Vec = "hello world ".repeat(100).into_bytes(); + let (raw_di, raw_digest) = sha256_digest(&raw); + let stored_zstd = zstd::bulk::compress(&raw, 19)?; + zstd_store + .update_zstd_oneshot( + raw_di, + DigestHasherFunc::Sha256, + Bytes::from(stored_zstd.clone()), + ) + .await?; + + // Incompressible blob: store it via the identity path so the physical zstd + // bytes are larger than the raw size. + let incompressible = incompressible_bytes(8192); + let (inc_di, inc_digest) = sha256_digest(&incompressible); + Store::new(zstd_store.clone()) + .update_oneshot(inc_di, Bytes::from(incompressible.clone())) + .await?; + + let response = cas_server + .batch_read_blobs(Request::new(BatchReadBlobsRequest { + instance_name: INSTANCE_NAME.to_string(), + digests: vec![raw_digest.clone(), inc_digest.clone()], + acceptable_compressors: vec![compressor::Value::Zstd.into()], + digest_function: digest_function::Value::Sha256.into(), + })) + .await? + .into_inner(); + + assert_eq!(response.responses.len(), 2); + let compressible = response + .responses + .iter() + .find(|r| r.digest.as_ref() == Some(&raw_digest)) + .expect("compressible response present"); + assert_eq!(compressible.compressor, compressor::Value::Zstd as i32); + // Byte-for-byte passthrough of the stored zstd stream, no recompression. + assert_eq!(compressible.data.as_ref(), stored_zstd.as_slice()); + assert_eq!( + zstd::bulk::decompress(&compressible.data, raw.len())?, + raw, + "stored zstd must decode to the original bytes" + ); + + let incompressible_resp = response + .responses + .iter() + .find(|r| r.digest.as_ref() == Some(&inc_digest)) + .expect("incompressible response present"); + assert_eq!( + incompressible_resp.compressor, + compressor::Value::Identity as i32, + "incompressible blob must not regress to a larger zstd response" + ); + assert_eq!(incompressible_resp.data.as_ref(), incompressible.as_slice()); + Ok(()) +} + +#[nativelink_test] +async fn batch_read_zstd_instance_error_isolation() -> Result<(), Box> { + let (cas_server, zstd_store) = make_zstd_instance_cas_server()?; + + let raw: Vec = "compress me ".repeat(64).into_bytes(); + let (raw_di, present_digest) = sha256_digest(&raw); + let stored_zstd = zstd::bulk::compress(&raw, 3)?; + zstd_store + .update_zstd_oneshot(raw_di, DigestHasherFunc::Sha256, Bytes::from(stored_zstd)) + .await?; + + let absent_digest = Digest { + hash: HASH3.to_string(), + size_bytes: 5, + }; + + let response = cas_server + .batch_read_blobs(Request::new(BatchReadBlobsRequest { + instance_name: INSTANCE_NAME.to_string(), + digests: vec![present_digest.clone(), absent_digest.clone()], + acceptable_compressors: vec![compressor::Value::Zstd.into()], + digest_function: digest_function::Value::Sha256.into(), + })) + .await? + .into_inner(); + + assert_eq!(response.responses.len(), 2); + let present = response + .responses + .iter() + .find(|r| r.digest.as_ref() == Some(&present_digest)) + .expect("present response"); + assert_eq!(present.status.as_ref().map(|s| s.code), Some(0)); + assert_eq!(present.compressor, compressor::Value::Zstd as i32); + + let absent = response + .responses + .iter() + .find(|r| r.digest.as_ref() == Some(&absent_digest)) + .expect("absent response"); + let status = absent.status.as_ref().expect("status set"); + assert_eq!(status.code, Code::NotFound as i32); + assert_eq!( + status.message, + format!( + "Key {:?} not found", + StoreKey::from(DigestInfo::try_from(absent_digest)?) + ), + "NotFound message should be trimmed to a single message" + ); + assert!(absent.data.is_empty()); + Ok(()) +} + +#[nativelink_test] +async fn batch_update_zstd_instance_roundtrip_and_corrupt_isolation() +-> Result<(), Box> { + let (cas_server, _zstd_store) = make_zstd_instance_cas_server()?; + + // Valid zstd entry. + let good_raw: Vec = "round trip ".repeat(80).into_bytes(); + let (_good_di, good_digest) = sha256_digest(&good_raw); + let good_compressed = zstd::bulk::compress(&good_raw, 3)?; + + // Corrupt entry: valid zstd of some bytes but with a digest whose hash does + // not match the content (decodes to the wrong hash). + let bad_raw = b"totally different content".to_vec(); + let bad_compressed = zstd::bulk::compress(&bad_raw, 3)?; + let bad_digest = Digest { + hash: HASH1.to_string(), + size_bytes: i64::try_from(bad_raw.len()).unwrap(), + }; + + let response = cas_server + .batch_update_blobs(Request::new(BatchUpdateBlobsRequest { + instance_name: INSTANCE_NAME.to_string(), + requests: vec![ + batch_update_blobs_request::Request { + digest: Some(good_digest.clone()), + data: good_compressed.into(), + compressor: compressor::Value::Zstd.into(), + }, + batch_update_blobs_request::Request { + digest: Some(bad_digest.clone()), + data: bad_compressed.into(), + compressor: compressor::Value::Zstd.into(), + }, + ], + digest_function: digest_function::Value::Sha256.into(), + })) + .await? + .into_inner(); + + assert_eq!(response.responses.len(), 2); + let good = response + .responses + .iter() + .find(|r| r.digest.as_ref() == Some(&good_digest)) + .expect("good response"); + assert_eq!( + good.status.as_ref().map(|s| s.code), + Some(0), + "valid zstd entry should succeed: {:?}", + good.status + ); + let bad = response + .responses + .iter() + .find(|r| r.digest.as_ref() == Some(&bad_digest)) + .expect("bad response"); + assert_eq!( + bad.status.as_ref().map(|s| s.code), + Some(Code::InvalidArgument as i32), + "corrupt entry should be InvalidArgument, got {:?}", + bad.status + ); + + // The valid entry round-trips byte-for-byte on read. + let read = cas_server + .batch_read_blobs(Request::new(BatchReadBlobsRequest { + instance_name: INSTANCE_NAME.to_string(), + digests: vec![good_digest.clone()], + acceptable_compressors: vec![compressor::Value::Identity.into()], + digest_function: digest_function::Value::Sha256.into(), + })) + .await? + .into_inner(); + assert_eq!(read.responses.len(), 1); + assert_eq!(read.responses[0].data.as_ref(), good_raw.as_slice()); + Ok(()) +} + +#[nativelink_test] +async fn batch_update_zstd_instance_identity_entry_roundtrips() +-> Result<(), Box> { + let (cas_server, _zstd_store) = make_zstd_instance_cas_server()?; + + let raw: Vec = "identity entry ".repeat(50).into_bytes(); + let (_di, digest) = sha256_digest(&raw); + + let response = cas_server + .batch_update_blobs(Request::new(BatchUpdateBlobsRequest { + instance_name: INSTANCE_NAME.to_string(), + requests: vec![batch_update_blobs_request::Request { + digest: Some(digest.clone()), + data: raw.clone().into(), + compressor: compressor::Value::Identity.into(), + }], + digest_function: digest_function::Value::Sha256.into(), + })) + .await? + .into_inner(); + assert_eq!(response.responses.len(), 1); + assert_eq!( + response.responses[0].status.as_ref().map(|s| s.code), + Some(0) + ); + + let read = cas_server + .batch_read_blobs(Request::new(BatchReadBlobsRequest { + instance_name: INSTANCE_NAME.to_string(), + digests: vec![digest.clone()], + acceptable_compressors: vec![compressor::Value::Identity.into()], + digest_function: digest_function::Value::Sha256.into(), + })) + .await? + .into_inner(); + assert_eq!(read.responses.len(), 1); + assert_eq!(read.responses[0].data.as_ref(), raw.as_slice()); + Ok(()) +} + +/// Builds the same zstd-compressing server with wire compression disabled. +fn make_zstd_instance_cas_server_compression_disabled() -> Result<(CasServer, Arc), Error> +{ + let temp_path = make_temp_path("cas_server_zstd_instance_disabled"); + std::fs::create_dir_all(&temp_path).expect("create temp dir"); + let inner = Store::new(MemoryStore::new(&MemorySpec::default())); + let zstd_store = ZstdStore::new(&zstd_instance_spec(temp_path), inner)?; + let store_manager = Arc::new(StoreManager::new()); + store_manager.add_store("main_cas", Store::new(zstd_store.clone()))?; + let cas_server = make_cas_server(&store_manager)?; + Ok((cas_server, zstd_store)) +} + +#[nativelink_test] +async fn batch_update_zstd_instance_rejected_when_remote_cache_compression_disabled() +-> Result<(), Box> { + // The wire-compression capability must not bypass the instance-level gate. + let (cas_server, _zstd_store) = make_zstd_instance_cas_server_compression_disabled()?; + + let raw_data = b"zstd disabled batch update to zstd-backed instance"; + let compressed_data = zstd::bulk::compress(raw_data, 3)?; + let digest = Digest { + hash: HASH1.to_string(), + size_bytes: i64::try_from(raw_data.len()).unwrap(), + }; + + let Err(status) = cas_server + .batch_update_blobs(Request::new(BatchUpdateBlobsRequest { + instance_name: INSTANCE_NAME.to_string(), + requests: vec![batch_update_blobs_request::Request { + digest: Some(digest), + data: compressed_data.into(), + compressor: compressor::Value::Zstd.into(), + }], + digest_function: digest_function::Value::Sha256.into(), + })) + .await + else { + panic!( + "zstd BatchUpdateBlobs to a ZstdStore-backed instance should fail when remote cache compression is disabled" + ); + }; + + assert_eq!(status.code(), Code::InvalidArgument); + assert!( + status + .message() + .contains("Remote cache compression is not supported"), + "unexpected error: {}", + status.message() + ); + + Ok(()) +} + +#[nativelink_test] +async fn batch_update_zstd_instance_identity_stores_when_remote_cache_compression_disabled() +-> Result<(), Box> { + // Identity entries must be unaffected by the gate: they still store and + // round-trip through the same ZstdStore-backed, compression-disabled + // instance. + let (cas_server, _zstd_store) = make_zstd_instance_cas_server_compression_disabled()?; + + let raw: Vec = "identity entry to disabled zstd instance " + .repeat(20) + .into_bytes(); + let (_di, digest) = sha256_digest(&raw); + + let response = cas_server + .batch_update_blobs(Request::new(BatchUpdateBlobsRequest { + instance_name: INSTANCE_NAME.to_string(), + requests: vec![batch_update_blobs_request::Request { + digest: Some(digest.clone()), + data: raw.clone().into(), + compressor: compressor::Value::Identity.into(), + }], + digest_function: digest_function::Value::Sha256.into(), + })) + .await? + .into_inner(); + assert_eq!(response.responses.len(), 1); + assert_eq!( + response.responses[0].status.as_ref().map(|s| s.code), + Some(0) + ); + + let read = cas_server + .batch_read_blobs(Request::new(BatchReadBlobsRequest { + instance_name: INSTANCE_NAME.to_string(), + digests: vec![digest.clone()], + acceptable_compressors: vec![compressor::Value::Identity.into()], + digest_function: digest_function::Value::Sha256.into(), + })) + .await? + .into_inner(); + assert_eq!(read.responses.len(), 1); + assert_eq!(read.responses[0].data.as_ref(), raw.as_slice()); + Ok(()) +} + const CHUNK1_VALUE: &str = "hello "; const CHUNK2_VALUE: &str = "world"; @@ -1081,11 +1461,11 @@ fn make_chunking_cas_server_with_avg( async fn upload_test_chunks(store: &Store) -> Result<(Digest, Digest, Digest), Error> { let chunk1_digest = Digest { hash: HASH1.to_string(), - size_bytes: CHUNK1_VALUE.len() as i64, + size_bytes: CHUNK1_VALUE.len().try_into().unwrap_or(i64::MAX), }; let chunk2_digest = Digest { hash: HASH2.to_string(), - size_bytes: CHUNK2_VALUE.len() as i64, + size_bytes: CHUNK2_VALUE.len().try_into().unwrap_or(i64::MAX), }; store .update_oneshot( @@ -1250,7 +1630,7 @@ async fn splice_blob_missing_chunk_returns_not_found() -> Result<(), Box Result<(), Box Result<(), Box Result<(), Box Result<(), Box Result<(), Box Result<(), Box Result<(), Box Result<(), Box> { - // avg 1024 (min allowed) with max_chunk_count 2: the 16 KiB test blob + // avg 1024 (min allowed) with `max_chunk_count` 2: the 16 KiB test blob // chunks to more than 2 pieces, so on-demand splitting must refuse. const AVG_CHUNK_SIZE: u64 = 1024; const BLOB_SIZE: usize = 16 * 1024; @@ -1641,7 +2023,7 @@ async fn max_chunk_count_limits_split_and_splice() -> Result<(), Box Result<(), Box) -> bool { + matches!(key, StoreKey::Digest(d) + if d.packed_hash() == self.stall_digest.packed_hash() + && d.size_bytes() == self.stall_digest.size_bytes()) + } +} + +#[async_trait] +impl StoreDriver for SelectiveStallStore { + async fn post_init(self: Arc) -> Result<(), Error> { + Ok(()) + } + + async fn has_with_results( + self: Pin<&Self>, + keys: &[StoreKey<'_>], + results: &mut [Option], + ) -> Result<(), Error> { + self.inner.has_with_results(keys, results).await + } + + async fn update( + self: Pin<&Self>, + key: StoreKey<'_>, + reader: DropCloserReadHalf, + size_info: UploadSizeInfo, + ) -> Result { + if self.stalls(&key) { + tokio::time::sleep(self.delay).await; + return Ok(0); + } + self.inner + .as_store_driver_pin() + .update(key, reader, size_info) + .await + } + + async fn get_part( + self: Pin<&Self>, + key: StoreKey<'_>, + writer: &mut DropCloserWriteHalf, + offset: u64, + length: Option, + ) -> Result<(), Error> { + if self.stalls(&key) { + tokio::time::sleep(self.delay).await; + return Ok(()); + } + self.inner + .as_store_driver_pin() + .get_part(key, writer, offset, length) + .await + } + + fn inner_store(&self, _key: Option) -> &dyn StoreDriver { + self + } + + fn as_any(&self) -> &(dyn core::any::Any + Sync + Send + 'static) { + self + } + + fn as_any_arc(self: Arc) -> Arc { + self + } + + fn register_remove_callback(self: Arc, _callback: RemoveCallback) -> Result<(), Error> { + Ok(()) + } +} + +default_health_status_indicator!(SelectiveStallStore); + +/// A batch update whose siblings are one healthy blob and one blob that stalls +/// past `BATCH_PER_BLOB_TIMEOUT` isolates the two: the healthy blob commits +/// (`Ok`), the stalled one fails with its own `DeadlineExceeded` status, and +/// neither outcome affects the other. +#[nativelink_test(start_paused = true)] +async fn batch_update_per_blob_timeout_isolates_siblings() -> Result<(), Box> +{ + let good_data = b"healthy sibling blob".to_vec(); + let (good_di, good_digest) = sha256_digest(&good_data); + let stall_data = b"stalling sibling blob".to_vec(); + let (stall_di, stall_digest) = sha256_digest(&stall_data); + + let inner = Store::new(MemoryStore::new(&MemorySpec::default())); + let store = Store::new(Arc::new(SelectiveStallStore { + stall_digest: stall_di, + delay: Duration::from_mins(2), // Longer than BATCH_PER_BLOB_TIMEOUT (30s). + inner: inner.clone(), + })); + let store_manager = Arc::new(StoreManager::new()); + store_manager.add_store("main_cas", store)?; + let cas_server = make_cas_server(&store_manager)?; + + let response = cas_server + .batch_update_blobs(Request::new(BatchUpdateBlobsRequest { + instance_name: INSTANCE_NAME.to_string(), + requests: vec![ + batch_update_blobs_request::Request { + digest: Some(good_digest.clone()), + data: good_data.clone().into(), + compressor: compressor::Value::Identity.into(), + }, + batch_update_blobs_request::Request { + digest: Some(stall_digest.clone()), + data: stall_data.into(), + compressor: compressor::Value::Identity.into(), + }, + ], + digest_function: digest_function::Value::Sha256.into(), + })) + .await + .unwrap() + .into_inner(); + + assert_eq!(response.responses.len(), 2); + let good = response + .responses + .iter() + .find(|r| r.digest.as_ref() == Some(&good_digest)) + .expect("healthy sibling response present"); + assert_eq!( + good.status.as_ref().map(|s| s.code), + Some(Code::Ok as i32), + "healthy sibling must commit despite the stalled one: {:?}", + good.status + ); + let stalled = response + .responses + .iter() + .find(|r| r.digest.as_ref() == Some(&stall_digest)) + .expect("stalled sibling response present"); + assert_eq!( + stalled.status.as_ref().map(|s| s.code), + Some(Code::DeadlineExceeded as i32), + "stalled sibling must surface its own DeadlineExceeded: {:?}", + stalled.status + ); + + // The healthy blob actually committed; the stalled one did not. + assert_eq!(inner.has(good_di).await?, Some(good_data.len() as u64)); + assert_eq!(inner.has(stall_di).await?, None); + Ok(()) +} diff --git a/nativelink-service/tests/grpc_wire_compression_test.rs b/nativelink-service/tests/grpc_wire_compression_test.rs new file mode 100644 index 000000000..8ded2f18f --- /dev/null +++ b/nativelink-service/tests/grpc_wire_compression_test.rs @@ -0,0 +1,552 @@ +// Copyright 2026 The NativeLink Authors. All rights reserved. +// +// Licensed under the Functional Source License, Version 1.1, Apache 2.0 Future License (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// See LICENSE file for details +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Integration tests for client-side wire compression in `GrpcStore` +//! (`GrpcSpec.experimental_remote_cache_compression`) against a real +//! compression-enabled `ByteStream`/CAS server. + +use core::sync::atomic::{AtomicU64, Ordering}; +use std::collections::HashMap; +use std::sync::Arc; + +use bytes::Bytes; +use futures::Future; +use http_body_util::BodyExt; +use hyper_util::rt::TokioIo; +use hyper_util::server::conn::auto; +use hyper_util::service::TowerToHyperService; +use nativelink_config::cas_server::{ByteStreamConfig, CasStoreConfig, WithInstanceName}; +use nativelink_config::stores::{GrpcEndpoint, GrpcSpec, MemorySpec, Retry, StoreType}; +use nativelink_error::{Code, Error}; +use nativelink_macro::nativelink_test; +use nativelink_proto::google::bytestream::{ + QueryWriteStatusRequest, QueryWriteStatusResponse, ReadRequest, ReadResponse, WriteRequest, + WriteResponse, +}; +use nativelink_service::bytestream_server::ByteStreamServer; +use nativelink_service::cas_server::CasServer; +use nativelink_service::wire_compression::RemoteCacheCompressionInstances; +use nativelink_store::grpc_store::GrpcStore; +use nativelink_store::memory_store::MemoryStore; +use nativelink_store::store_manager::StoreManager; +use nativelink_util::background_spawn; +use nativelink_util::common::DigestInfo; +use nativelink_util::store_trait::{Store, StoreLike}; +use sha2::{Digest as ShaDigest, Sha256}; + +const INSTANCE_NAME: &str = ""; + +#[derive(Clone)] +struct Executor; +impl hyper::rt::Executor for Executor +where + F: Future + Send + 'static, + F::Output: Send + 'static, +{ + fn execute(&self, fut: F) { + background_spawn!("test_executor", fut); + } +} + +async fn start_real_cas_server( + memory_store: Arc, + server_compression_enabled: bool, +) -> Result { + let store_manager = Arc::new(StoreManager::new()); + store_manager.add_store("main_cas", Store::new(memory_store))?; + let compression_instances = if server_compression_enabled { + RemoteCacheCompressionInstances::from_enabled_instance_names([INSTANCE_NAME.to_string()]) + } else { + RemoteCacheCompressionInstances::default() + }; + + let bs_server = ByteStreamServer::new( + &[WithInstanceName { + instance_name: INSTANCE_NAME.to_string(), + config: ByteStreamConfig { + cas_store: "main_cas".to_string(), + ..Default::default() + }, + }], + &store_manager, + &compression_instances, + )?; + let cas_server = CasServer::new( + &[WithInstanceName { + instance_name: INSTANCE_NAME.to_string(), + config: CasStoreConfig { + cas_store: "main_cas".to_string(), + experimental_chunking: None, + }, + }], + &store_manager, + &compression_instances, + )?; + + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let port = listener.local_addr().unwrap().port(); + + let routes = tonic::service::Routes::new(bs_server.into_service()) + .add_service(cas_server.into_service()); + let adapted_service = tower::ServiceBuilder::new() + .map_request(|req: hyper::Request| { + let (parts, body) = req.into_parts(); + let body = body + .map_err(|e| tonic::Status::internal(e.to_string())) + .boxed_unsync(); + hyper::Request::from_parts(parts, body) + }) + .service(routes); + let hyper_service = TowerToHyperService::new(adapted_service); + + background_spawn!("test_server_accept", async move { + loop { + let Ok((stream, _addr)) = listener.accept().await else { + break; + }; + stream.set_nodelay(true).unwrap(); + let hyper_service = hyper_service.clone(); + background_spawn!("test_server_conn", async move { + drop( + auto::Builder::new(Executor) + .serve_connection_with_upgrades(TokioIo::new(stream), hyper_service) + .await, + ); + }); + } + }); + Ok(port) +} + +fn grpc_spec(port: u16, compression: bool) -> GrpcSpec { + GrpcSpec { + instance_name: INSTANCE_NAME.to_string(), + endpoints: vec![GrpcEndpoint { + address: format!("http://127.0.0.1:{port}"), + tls_config: None, + concurrency_limit: None, + connect_timeout_s: 0, + tcp_keepalive_s: 0, + http2_keepalive_interval_s: 0, + http2_keepalive_timeout_s: 0, + }], + store_type: StoreType::Cas, + retry: Retry::default(), + max_concurrent_requests: 0, + connections_per_endpoint: 0, + rpc_timeout_s: 120, + use_legacy_resource_names: false, + headers: HashMap::new(), + forward_headers: vec![], + experimental_read_batching: None, + experimental_remote_cache_compression: compression, + } +} + +/// Structured, ~3:1-compressible content with a real sha256 digest (both +/// wire directions digest-verify). +fn make_content(tag: u8, len: usize) -> (Bytes, DigestInfo) { + let mut data = vec![0u8; len]; + for (i, b) in data.iter_mut().enumerate() { + *b = match i % 8 { + 0..=4 => tag, + 5 => u8::try_from((i / 8) % 256).expect("value is reduced modulo 256"), + _ => u8::try_from((i / 4096) % 256).expect("value is reduced modulo 256"), + }; + } + let mut hasher = Sha256::new(); + hasher.update(&data); + let digest = DigestInfo::try_new(&hex::encode(hasher.finalize()), len).unwrap(); + (Bytes::from(data), digest) +} + +// A blob above the compression threshold uploads via compressed-blobs and +// must land byte-identical (the server decodes and digest-verifies). +#[nativelink_test] +async fn compressed_upload_round_trip() -> Result<(), Box> { + let memory_store = MemoryStore::new(&MemorySpec::default()); + let port = start_real_cas_server(memory_store.clone(), true).await?; + let grpc_store = GrpcStore::new(&grpc_spec(port, true)).await?; + + let (content, digest) = make_content(0xA1, 1024 * 1024); + grpc_store.update_oneshot(digest, content.clone()).await?; + + let stored = memory_store.get_part_unchunked(digest, 0, None).await?; + assert_eq!(stored, content, "decoded upload must be byte-identical"); + Ok(()) +} + +// A full-blob read above the threshold downloads via compressed-blobs; the +// client decodes and digest-verifies. +#[nativelink_test] +async fn compressed_download_round_trip() -> Result<(), Box> { + let memory_store = MemoryStore::new(&MemorySpec::default()); + let port = start_real_cas_server(memory_store.clone(), true).await?; + let grpc_store = GrpcStore::new(&grpc_spec(port, true)).await?; + + let (content, digest) = make_content(0xB2, 1024 * 1024); + memory_store.update_oneshot(digest, content.clone()).await?; + + let fetched = grpc_store.get_part_unchunked(digest, 0, None).await?; + assert_eq!(fetched, content, "decoded download must be byte-identical"); + Ok(()) +} + +// Blobs below the threshold must use the identity path: they round-trip +// even against a server with compression disabled. +#[nativelink_test] +async fn small_blob_uses_identity_path() -> Result<(), Box> { + let memory_store = MemoryStore::new(&MemorySpec::default()); + let port = start_real_cas_server(memory_store.clone(), false).await?; + let grpc_store = GrpcStore::new(&grpc_spec(port, true)).await?; + + let (content, digest) = make_content(0xC3, 4 * 1024); + grpc_store.update_oneshot(digest, content.clone()).await?; + let fetched = grpc_store.get_part_unchunked(digest, 0, None).await?; + assert_eq!(fetched, content); + Ok(()) +} + +// Ranged reads must use the identity path even above the threshold: they +// round-trip against a compression-disabled server. +#[nativelink_test] +async fn ranged_read_uses_identity_path() -> Result<(), Box> { + let memory_store = MemoryStore::new(&MemorySpec::default()); + let compressed_port = start_real_cas_server(memory_store.clone(), true).await?; + let grpc_store = GrpcStore::new(&grpc_spec(compressed_port, true)).await?; + + let (content, digest) = make_content(0xD4, 1024 * 1024); + memory_store.update_oneshot(digest, content.clone()).await?; + + let fetched = grpc_store + .get_part_unchunked(digest, 100, Some(1000)) + .await?; + assert_eq!(fetched, content.slice(100..1100)); + Ok(()) +} + +// A compressed upload to an upstream without compression enabled fails with +// InvalidArgument (config-trust contract, documented on the field). +#[nativelink_test] +async fn upload_to_non_compressed_upstream_fails() -> Result<(), Box> { + let memory_store = MemoryStore::new(&MemorySpec::default()); + let port = start_real_cas_server(memory_store.clone(), false).await?; + let grpc_store = GrpcStore::new(&grpc_spec(port, true)).await?; + + let (content, digest) = make_content(0xE5, 1024 * 1024); + let err = grpc_store + .update_oneshot(digest, content) + .await + .expect_err("expected compressed upload to fail against plain upstream"); + assert_eq!(err.code, Code::InvalidArgument, "unexpected error: {err:?}"); + Ok(()) +} + +// Flag off: byte-identical legacy behavior against a plain upstream. +#[nativelink_test] +async fn flag_off_plain_round_trip() -> Result<(), Box> { + let memory_store = MemoryStore::new(&MemorySpec::default()); + let port = start_real_cas_server(memory_store.clone(), false).await?; + let grpc_store = GrpcStore::new(&grpc_spec(port, false)).await?; + + let (content, digest) = make_content(0xF6, 1024 * 1024); + grpc_store.update_oneshot(digest, content.clone()).await?; + let fetched = grpc_store.get_part_unchunked(digest, 0, None).await?; + assert_eq!(fetched, content); + Ok(()) +} + +// --------------------------------------------------------------------------- +// Review-round regression tests: W1 (no resume storm), W2 (corruption is +// terminal), W3 (no hang on early completion). These need a controllable +// fake ByteStream server rather than the real one. +// --------------------------------------------------------------------------- + +#[derive(Debug, Default)] +struct FakeByteStream { + write_calls: Arc, + identity_reads: Arc, + /// Write behavior: consume this many messages, then abort the stream + /// with UNAVAILABLE. Zero means accept-one-then-early-complete (W3). + abort_write_after_messages: Option, + early_complete_write: bool, + /// Read behavior: payload served for compressed-blobs reads. + compressed_read_payload: Option, + /// Pause the compressed read stream after this many chunks (forces the + /// client feed to lag the decoder, exercising the cascade race). + pause_compressed_read_after_chunks: Option, + /// Hold the compressed read stream open forever after this many chunks. + /// This ensures a decoder error must cancel the feed rather than waiting + /// for the upstream stream to finish. + hold_compressed_read_after_chunks: Option, + /// Payload served for identity reads (fallback detector). + identity_read_payload: Option, +} + +type ReadStreamT = + core::pin::Pin> + Send>>; + +#[tonic::async_trait] +impl nativelink_proto::google::bytestream::byte_stream_server::ByteStream for FakeByteStream { + type ReadStream = ReadStreamT; + + async fn read( + &self, + request: tonic::Request, + ) -> Result, tonic::Status> { + let resource_name = request.into_inner().resource_name; + if resource_name.contains("compressed-blobs") { + let payload = self + .compressed_read_payload + .clone() + .ok_or_else(|| tonic::Status::not_found("no compressed payload configured"))?; + let chunks: Vec = payload + .chunks(64 * 1024) + .map(Bytes::copy_from_slice) + .collect(); + let pause_after = self.pause_compressed_read_after_chunks; + let hold_after = self.hold_compressed_read_after_chunks; + let stream = + futures::stream::unfold((chunks, 0usize), move |(chunks, index)| async move { + if index >= chunks.len() { + return None; + } + if Some(index) == hold_after { + futures::future::pending::<()>().await; + } + if Some(index) == pause_after { + // Hold the wire open long enough that the client + // decoder settles first. + tokio::time::sleep(core::time::Duration::from_millis(500)).await; + } + let data = chunks[index].clone(); + Some((Ok(ReadResponse { data }), (chunks, index + 1))) + }); + let boxed: Self::ReadStream = Box::pin(stream); + return Ok(tonic::Response::new(boxed)); + } + self.identity_reads.fetch_add(1, Ordering::Relaxed); + let payload = self + .identity_read_payload + .clone() + .ok_or_else(|| tonic::Status::not_found("no identity payload configured"))?; + let chunks: Vec> = payload + .chunks(64 * 1024) + .map(|c| { + Ok(ReadResponse { + data: Bytes::copy_from_slice(c), + }) + }) + .collect(); + let boxed: Self::ReadStream = Box::pin(futures::stream::iter(chunks)); + Ok(tonic::Response::new(boxed)) + } + + async fn write( + &self, + request: tonic::Request>, + ) -> Result, tonic::Status> { + self.write_calls.fetch_add(1, Ordering::Relaxed); + let mut stream = request.into_inner(); + if self.early_complete_write { + // Read exactly one message, then settle the write early per the + // REAPI duplicate-upload contract without consuming the rest. + let _first = stream.message().await?; + return Ok(tonic::Response::new(WriteResponse { committed_size: -1 })); + } + if let Some(abort_after) = self.abort_write_after_messages { + let mut seen = 0u64; + while seen < abort_after { + match stream.message().await? { + Some(_msg) => seen += 1, + None => break, + } + } + return Err(tonic::Status::unavailable("injected mid-stream failure")); + } + Err(tonic::Status::unimplemented("no write behavior configured")) + } + + async fn query_write_status( + &self, + _request: tonic::Request, + ) -> Result, tonic::Status> { + Err(tonic::Status::unimplemented("not used in these tests")) + } +} + +async fn start_fake_bytestream_server(fake: FakeByteStream) -> Result { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let port = listener.local_addr().unwrap().port(); + let routes = tonic::service::Routes::new( + nativelink_proto::google::bytestream::byte_stream_server::ByteStreamServer::new(fake), + ); + let adapted_service = tower::ServiceBuilder::new() + .map_request(|req: hyper::Request| { + let (parts, body) = req.into_parts(); + let body = body + .map_err(|e| tonic::Status::internal(e.to_string())) + .boxed_unsync(); + hyper::Request::from_parts(parts, body) + }) + .service(routes); + let hyper_service = TowerToHyperService::new(adapted_service); + background_spawn!("fake_server_accept", async move { + loop { + let Ok((stream, _addr)) = listener.accept().await else { + break; + }; + stream.set_nodelay(true).unwrap(); + let hyper_service = hyper_service.clone(); + background_spawn!("fake_server_conn", async move { + drop( + auto::Builder::new(Executor) + .serve_connection_with_upgrades(TokioIo::new(stream), hyper_service) + .await, + ); + }); + } + }); + Ok(port) +} + +// W1: a mid-stream failure of a compressed upload must fail fast with the +// transport error — exactly one write attempt, no resume storm ending in +// InvalidArgument. +#[nativelink_test] +async fn compressed_upload_mid_stream_failure_fails_fast() -> Result<(), Box> +{ + let write_calls = Arc::new(AtomicU64::new(0)); + let port = start_fake_bytestream_server(FakeByteStream { + write_calls: write_calls.clone(), + abort_write_after_messages: Some(2), + ..Default::default() + }) + .await?; + let mut spec = grpc_spec(port, true); + spec.retry = Retry { + max_retries: 3, + delay: 0.01, + ..Default::default() + }; + let grpc_store = GrpcStore::new(&spec).await?; + + let (content, digest) = make_content(0x11, 4 * 1024 * 1024); + let result = grpc_store.update_oneshot(digest, content).await; + let err = result.expect_err("expected mid-stream failure to surface"); + assert_eq!( + err.code, + Code::Unavailable, + "must surface the transport error, not a resume-replay InvalidArgument: {err:?}" + ); + assert_eq!( + write_calls.load(Ordering::Relaxed), + 1, + "compressed uploads must not replay-resume after partial consumption" + ); + Ok(()) +} + +// W2: decoder-detected corruption mid-stream must be terminal +// InvalidArgument deterministically — never silently retried via the +// unverified identity path. +#[nativelink_test] +async fn corrupt_compressed_download_is_terminal() -> Result<(), Box> { + // The decoder must abort MID-STREAM while the feed still has chunks in + // flight (the old classification then saw a cascade "receiver + // disconnected" feed error and silently fell back to the unverified + // identity path). Bit-flips inside a zstd block decode to garbage and + // only fail the digest check at EOF, so instead append a second valid + // frame: its first decoded block overshoots the expected size, which + // the decoder rejects immediately and deterministically. + use nativelink_proto::build::bazel::remote::execution::v2::compressor; + let (content, digest) = make_content(0x22, 8 * 1024 * 1024); + let valid_frame = + nativelink_util::wire_compression::compress(content.clone(), compressor::Value::Zstd)?; + let (extra, _) = make_content(0x99, 8 * 1024 * 1024); + let overflow_frame = + nativelink_util::wire_compression::compress(extra, compressor::Value::Zstd)?; + let valid_frame_chunks = valid_frame.len().div_ceil(64 * 1024); + let mut corrupt = valid_frame.to_vec(); + corrupt.extend_from_slice(&overflow_frame); + let identity_reads = Arc::new(AtomicU64::new(0)); + let port = start_fake_bytestream_server(FakeByteStream { + identity_reads: identity_reads.clone(), + // Pause just after the first frame ends: the decoder hits the + // size overshoot while the feed is still blocked on the wire, so + // the feed's next send hits the dropped receiver. + hold_compressed_read_after_chunks: Some(valid_frame_chunks + 1), + compressed_read_payload: Some(Bytes::from(corrupt)), + identity_read_payload: Some(content), + ..Default::default() + }) + .await?; + let grpc_store = GrpcStore::new(&grpc_spec(port, true)).await?; + + let result = tokio::time::timeout( + core::time::Duration::from_secs(2), + grpc_store.get_part_unchunked(digest, 0, None), + ) + .await + .expect("decoder failure must cancel the held compressed read stream"); + let err = result.expect_err("corrupt compressed data must be terminal"); + assert_eq!( + err.code, + Code::InvalidArgument, + "decoder verdict must win over cascade transport classification: {err:?}" + ); + assert_eq!( + identity_reads.load(Ordering::Relaxed), + 0, + "corrupt data must never fall back to the unverified identity path" + ); + Ok(()) +} + +// W3: a server that early-completes a compressed upload while the worker +// producer is still sending must not turn the producer's send/drain into an +// error. +#[nativelink_test] +async fn compressed_upload_returns_after_early_completion() +-> Result<(), Box> { + use nativelink_util::buf_channel::make_buf_channel_pair; + use nativelink_util::store_trait::UploadSizeInfo; + + let port = start_fake_bytestream_server(FakeByteStream { + early_complete_write: true, + ..Default::default() + }) + .await?; + let grpc_store = GrpcStore::new(&grpc_spec(port, true)).await?; + + let (content, digest) = make_content(0x33, 4 * 1024 * 1024); + let (mut tx, rx) = make_buf_channel_pair(); + let producer_content = content.clone(); + let producer = async move { + for chunk in producer_content.chunks(64 * 1024) { + tx.send(Bytes::copy_from_slice(chunk)).await?; + } + tx.send_eof() + }; + let update_fut = grpc_store.update(digest, rx, UploadSizeInfo::ExactSize(content.len() as u64)); + let (result, producer_result) = + tokio::time::timeout(core::time::Duration::from_secs(10), async { + tokio::join!(update_fut, producer) + }) + .await + .expect("early-completed compressed upload must not hang the producer"); + result?; + producer_result?; + Ok(()) +} diff --git a/nativelink-service/tests/wire_compression_test.rs b/nativelink-service/tests/wire_compression_test.rs index f829c7dfa..8463943cf 100644 --- a/nativelink-service/tests/wire_compression_test.rs +++ b/nativelink-service/tests/wire_compression_test.rs @@ -15,20 +15,20 @@ //! Unit tests for the REAPI wire-compression helpers shared by the //! `ByteStream` and CAS services: compressor URI resolution, zstd //! encode/decode with size validation, the zero-copy identity batch-update -//! path, and the blocking [`BufChannelReader`] adapter used by the streaming -//! zstd encoder/decoder. +//! path, and blocking-pool isolation of the async streaming zstd encoder. -use std::io::Read; +use core::time::Duration; use bytes::Bytes; use nativelink_error::{Code, Error}; use nativelink_macro::nativelink_test; use nativelink_proto::build::bazel::remote::execution::v2::compressor; use nativelink_service::wire_compression::{ - BufChannelReader, compress, decompress, decompress_batch_update, resolve_wire_compressor, + ZSTD_COMPRESSION_LEVEL, compress, decompress, decompress_batch_update, resolve_wire_compressor, + stream_encode_compressed_download, }; use nativelink_util::buf_channel::make_buf_channel_pair; -use nativelink_util::spawn_blocking; +use nativelink_util::{spawn, spawn_blocking}; use pretty_assertions::assert_eq; #[nativelink_test] @@ -164,38 +164,107 @@ async fn resolve_wire_compressor_accepts_only_advertised_compressors() -> Result Ok(()) } -#[nativelink_test] -async fn buf_channel_reader_reads_partial_and_multiple_chunks() -> Result<(), Error> { - let (mut tx, rx) = make_buf_channel_pair(); - - // BufChannelReader is a blocking `Read` adapter (it uses - // `blocking_recv`), so it must run off the async runtime — exactly how - // the streaming zstd encoder/decoder drive it in production. - let reader_task = spawn_blocking!("buf_channel_reader_test", move || { - let mut reader = BufChannelReader::new(rx); - - let mut first_partial_chunk = [0; 2]; - reader - .read_exact(&mut first_partial_chunk) - .expect("failed to read first partial chunk"); - assert_eq!(&first_partial_chunk, b"ab"); - - let mut remaining_chunks = [0; 6]; - reader - .read_exact(&mut remaining_chunks) - .expect("failed to read remaining chunks"); - assert_eq!(&remaining_chunks, b"cdefgh"); - - let mut eof_probe = [0; 1]; - assert_eq!(reader.read(&mut eof_probe).expect("failed to read EOF"), 0); - }); +/// A compressed-download encode stream lives at the gRPC CLIENT's drain rate: +/// its output channel only empties as fast as the client reads, and a blob can +/// take arbitrarily long to stream out. If each such stream occupies a tokio +/// blocking-pool thread for its whole lifetime, N concurrent downloads with +/// slow consumers occupy N threads and every unrelated `spawn_blocking` user +/// (filesystem store I/O, upload decode, credential providers) queues behind +/// streams that may not finish for minutes. This test wires up encode streams +/// exactly the way `ByteStreamServer::inner_read_compressed` does, with more +/// held streams than blocking-pool threads, and asserts that a trivial +/// unrelated `spawn_blocking` closure still gets to run. +#[test] +#[expect( + clippy::disallowed_methods, + reason = "the defect under test is blocking-pool capacity, so the test needs \ + a dedicated runtime with a deterministically small blocking pool" +)] +fn held_compressed_download_streams_must_not_starve_blocking_pool() { + // More encode streams than blocking-pool threads. The streams never + // complete during the test: their input never reaches EOF and their + // consumer never reads, so any per-stream thread is held indefinitely. + // The blocking pool schedules FIFO, so if each stream occupies a thread + // the sentinel spawned afterwards can never run — no sleeps or timing + // races are needed for the starvation to be deterministic. + const MAX_BLOCKING_THREADS: usize = 4; + const NUM_STREAMS: usize = 8; + const SENTINEL_TIMEOUT: Duration = Duration::from_secs(2); + const OVERALL_TIMEOUT: Duration = Duration::from_secs(30); + + let runtime = tokio::runtime::Builder::new_multi_thread() + .worker_threads(2) + .max_blocking_threads(MAX_BLOCKING_THREADS) + .enable_all() + .build() + .expect("failed to build test runtime"); + + runtime.block_on(async { + tokio::time::timeout(OVERALL_TIMEOUT, async { + // Held alive for the duration of the sentinel probe: dropping the + // feeders, encode handles, or receivers early would tear the + // streams down and release any held threads. + let mut held_streams = Vec::with_capacity(NUM_STREAMS); + for _ in 0..NUM_STREAMS { + let (mut raw_tx, raw_rx) = make_buf_channel_pair(); + let (compressed_tx, compressed_rx) = make_buf_channel_pair(); + + // Feed incompressible pseudo-random data forever (no EOF) so + // the encoder always has input and keeps producing output. + // The output channel has a small fixed capacity and nothing + // reads `compressed_rx`, so the encoder soon waits on a full + // output channel — the "slow client" that holds the stream + // open. Async sends here mean the feeders themselves never + // occupy blocking-pool threads. + let feeder = spawn!("test_raw_data_feeder", async move { + let mut state = 0x9E37_79B9_7F4A_7C15_u64; + loop { + let mut chunk = vec![0u8; 64 * 1024]; + for word in chunk.chunks_exact_mut(8) { + state = state + .wrapping_mul(6_364_136_223_846_793_005) + .wrapping_add(1_442_695_040_888_963_407); + word.copy_from_slice(&state.to_le_bytes()); + } + if raw_tx.send(Bytes::from(chunk)).await.is_err() { + break; + } + } + }); - tx.send(Bytes::from_static(b"abc")).await?; - tx.send(Bytes::from_static(b"defgh")).await?; - tx.send_eof()?; + // Start the encode stream the same way + // `ByteStreamServer::inner_read_compressed` does: as a plain + // async future on the runtime, never on the blocking pool. + let encode_task = spawn!( + "test_encode_compressed_download", + stream_encode_compressed_download( + raw_rx, + compressor::Value::Zstd, + ZSTD_COMPRESSION_LEVEL, + compressed_tx, + ) + ); - reader_task + held_streams.push((feeder, encode_task, compressed_rx)); + } + + // Unrelated blocking work must not starve behind held + // compressed-download streams. + let sentinel = spawn_blocking!("test_blocking_pool_sentinel", || ()); + let sentinel_result = tokio::time::timeout(SENTINEL_TIMEOUT, sentinel).await; + assert!( + sentinel_result.is_ok(), + "unrelated blocking work must not starve behind {NUM_STREAMS} held \ + compressed-download streams on a {MAX_BLOCKING_THREADS}-thread blocking pool" + ); + + drop(held_streams); + }) .await - .map_err(|e| Error::new(Code::Internal, format!("reader task panicked: {e:?}")))?; - Ok(()) + .expect("test exceeded overall timeout"); + }); + + // Encode streams stuck on a full output channel park their blocking-pool + // threads indefinitely; a normal runtime drop would join them and hang. + runtime.shutdown_background(); } diff --git a/nativelink-store/BUILD.bazel b/nativelink-store/BUILD.bazel index 11a2ca865..157d6fb2b 100644 --- a/nativelink-store/BUILD.bazel +++ b/nativelink-store/BUILD.bazel @@ -49,6 +49,7 @@ rust_library( "src/size_partitioning_store.rs", "src/store_manager.rs", "src/verify_store.rs", + "src/zstd_store.rs", ], proc_macro_deps = [ "@crates//:async-trait", @@ -111,7 +112,11 @@ rust_library( "@crates//:uuid", "@crates//:webpki-roots", "@crates//:wincode", - ], + "@crates//:zstd", + ] + select({ + "@platforms//os:macos": ["@crates//:libc"], + "//conditions:default": [], + }), ) rust_test_suite( @@ -132,6 +137,7 @@ rust_test_suite( "tests/filesystem_store_test.rs", "tests/gcs_client_test.rs", "tests/gcs_store_test.rs", + "tests/grpc_read_batching_test.rs", "tests/grpc_store_test.rs", "tests/memory_store_test.rs", "tests/mongo_store_test.rs", @@ -146,6 +152,7 @@ rust_test_suite( "tests/size_partitioning_store_test.rs", "tests/store_manager_test.rs", "tests/verify_store_test.rs", + "tests/zstd_store_test.rs", ], compile_data = [ "tests/mongo_runner/mod.rs", @@ -207,6 +214,7 @@ rust_test_suite( "@crates//:uuid", "@crates//:wincode", "@crates//:zip", + "@crates//:zstd", ], ) diff --git a/nativelink-store/Cargo.toml b/nativelink-store/Cargo.toml index 538b4a083..fed2778ab 100644 --- a/nativelink-store/Cargo.toml +++ b/nativelink-store/Cargo.toml @@ -4,7 +4,7 @@ lints.workspace = true [package] edition = "2024" name = "nativelink-store" -version = "1.6.1" +version = "1.6.3" [dependencies] nativelink-config = { path = "../nativelink-config" } @@ -130,6 +130,10 @@ wincode = { version = "0.5.4", default-features = false, features = [ "alloc", "derive", ] } +zstd = { version = "0.13.3", default-features = false } + +[target.'cfg(target_os = "macos")'.dependencies] +libc = { version = "0.2.177", default-features = false } [dev-dependencies] nativelink-macro = { path = "../nativelink-macro" } diff --git a/nativelink-store/src/azure_blob_store.rs b/nativelink-store/src/azure_blob_store.rs index 194336c81..c01627d1c 100644 --- a/nativelink-store/src/azure_blob_store.rs +++ b/nativelink-store/src/azure_blob_store.rs @@ -42,7 +42,7 @@ use nativelink_util::health_utils::{HealthRegistryBuilder, HealthStatus, HealthS use nativelink_util::instant_wrapper::InstantWrapper; use nativelink_util::retry::{Retrier, RetryResult}; use nativelink_util::store_trait::{ - RemoveItemCallback, StoreDriver, StoreKey, StoreOptimizations, UploadSizeInfo, + RemoveCallback, StoreDriver, StoreKey, StoreOptimizations, UploadSizeInfo, }; use tokio::sync::mpsc; use tokio::time::sleep; @@ -242,7 +242,10 @@ where if self.consider_expired_after_s > 0 && let Some(last_modified) = props.last_modified().ok().flatten() { - let now = (self.now_fn)().unix_timestamp() as i64; + let now = (self.now_fn)() + .unix_timestamp() + .try_into() + .unwrap_or(i64::MAX); if last_modified.unix_timestamp() + self.consider_expired_after_s <= now { @@ -720,10 +723,7 @@ where registry.register_indicator(self); } - fn register_remove_callback( - self: Arc, - _callback: Arc, - ) -> Result<(), Error> { + fn register_remove_callback(self: Arc, _callback: RemoveCallback) -> Result<(), Error> { // Azure Blob Storage manages object lifecycle externally, // so we can safely ignore remove callbacks. Ok(()) diff --git a/nativelink-store/src/cache_metrics_store.rs b/nativelink-store/src/cache_metrics_store.rs index 9c3a7933a..deca501d6 100644 --- a/nativelink-store/src/cache_metrics_store.rs +++ b/nativelink-store/src/cache_metrics_store.rs @@ -31,7 +31,7 @@ use nativelink_util::fs; use nativelink_util::health_utils::{HealthRegistryBuilder, HealthStatusIndicator}; use nativelink_util::metrics::{CACHE_METRICS, CACHE_TYPE, CacheMetricAttrs}; use nativelink_util::store_trait::{ - RemoveItemCallback, Store, StoreDriver, StoreKey, StoreLike, StoreOptimizations, UploadSizeInfo, + RemoveCallback, Store, StoreDriver, StoreKey, StoreLike, StoreOptimizations, UploadSizeInfo, }; use opentelemetry::KeyValue; @@ -275,10 +275,7 @@ impl StoreDriver for CacheMetricsStore { self.backend.clone().register_health(registry); } - fn register_remove_callback( - self: Arc, - callback: Arc, - ) -> Result<(), Error> { + fn register_remove_callback(self: Arc, callback: RemoveCallback) -> Result<(), Error> { self.backend.register_remove_callback(callback) } } diff --git a/nativelink-store/src/callback_utils.rs b/nativelink-store/src/callback_utils.rs index a18f20c52..29e0783c7 100644 --- a/nativelink-store/src/callback_utils.rs +++ b/nativelink-store/src/callback_utils.rs @@ -14,25 +14,24 @@ use core::borrow::Borrow; use core::pin::Pin; -use std::sync::Arc; use nativelink_util::evicting_map; -use nativelink_util::store_trait::{RemoveItemCallback, StoreKey}; +use nativelink_util::store_trait::{RemoveCallback, StoreKey}; -// Generic struct to hold a RemoveItemCallback ref for the purposes +// Generic struct to hold a RemoveCallback ref for the purposes // of a RemoveStateCallback call #[derive(Debug)] -pub struct RemoveItemCallbackHolder { - callback: Arc, +pub struct RemoveCallbackHolder { + callback: RemoveCallback, } -impl RemoveItemCallbackHolder { - pub fn new(callback: Arc) -> Self { +impl RemoveCallbackHolder { + pub fn new(callback: RemoveCallback) -> Self { Self { callback } } } -impl<'a, Q> evicting_map::RemoveItemCallback for RemoveItemCallbackHolder +impl<'a, Q> evicting_map::RemoveItemCallback for RemoveCallbackHolder where Q: Borrow>, { diff --git a/nativelink-store/src/completeness_checking_store.rs b/nativelink-store/src/completeness_checking_store.rs index a7f81f807..38e054cef 100644 --- a/nativelink-store/src/completeness_checking_store.rs +++ b/nativelink-store/src/completeness_checking_store.rs @@ -29,7 +29,7 @@ use nativelink_util::common::DigestInfo; use nativelink_util::health_utils::{HealthStatusIndicator, default_health_status_indicator}; use nativelink_util::metrics_utils::CounterWithTime; use nativelink_util::store_trait::{ - RemoveItemCallback, Store, StoreDriver, StoreKey, StoreLike, UploadSizeInfo, + RemoveCallback, Store, StoreDriver, StoreKey, StoreLike, UploadSizeInfo, }; use parking_lot::Mutex; use tokio::sync::Notify; @@ -398,10 +398,7 @@ impl StoreDriver for CompletenessCheckingStore { self } - fn register_remove_callback( - self: Arc, - callback: Arc, - ) -> Result<(), Error> { + fn register_remove_callback(self: Arc, callback: RemoveCallback) -> Result<(), Error> { self.ac_store.register_remove_callback(callback.clone())?; self.cas_store.register_remove_callback(callback)?; Ok(()) diff --git a/nativelink-store/src/compression_store.rs b/nativelink-store/src/compression_store.rs index 1bf816c7b..b291949dc 100644 --- a/nativelink-store/src/compression_store.rs +++ b/nativelink-store/src/compression_store.rs @@ -22,7 +22,7 @@ use bytes::{Buf, BufMut, BytesMut}; use futures::future::FutureExt; use lz4_flex::block::{compress_into, decompress_into, get_maximum_output_size}; use nativelink_config::stores::CompressionSpec; -use nativelink_error::{Code, Error, ResultExt, error_if, make_err}; +use nativelink_error::{Code, Error, ResultExt, error_if, make_err, make_input_err}; use nativelink_metric::MetricsComponent; use nativelink_util::buf_channel::{ DropCloserReadHalf, DropCloserWriteHalf, make_buf_channel_pair, @@ -30,7 +30,7 @@ use nativelink_util::buf_channel::{ use nativelink_util::health_utils::{HealthStatusIndicator, default_health_status_indicator}; use nativelink_util::spawn; use nativelink_util::store_trait::{ - RemoveItemCallback, Store, StoreDriver, StoreKey, StoreLike, UploadSizeInfo, + RemoveCallback, Store, StoreDriver, StoreKey, StoreLike, UploadSizeInfo, }; use serde::{Deserialize, Serialize}; use wincode::{SchemaRead, SchemaWrite}; @@ -257,8 +257,9 @@ impl core::fmt::Debug for CompressionStore { impl CompressionStore { pub fn new(spec: &CompressionSpec, inner_store: Store) -> Result, Error> { - let lz4_config = match spec.compression_algorithm { - nativelink_config::stores::CompressionAlgorithm::Lz4(mut lz4_config) => { + let lz4_config = match &spec.compression_algorithm { + nativelink_config::stores::CompressionAlgorithm::Lz4(lz4_config) => { + let mut lz4_config = *lz4_config; if lz4_config.block_size == 0 { lz4_config.block_size = DEFAULT_BLOCK_SIZE; } @@ -267,6 +268,11 @@ impl CompressionStore { } lz4_config } + nativelink_config::stores::CompressionAlgorithm::Zstd(_) => { + return Err(make_input_err!( + "CompressionStore only implements the lz4 compression algorithm" + )); + } }; Ok(Arc::new(Self { inner_store, @@ -708,10 +714,7 @@ impl StoreDriver for CompressionStore { self } - fn register_remove_callback( - self: Arc, - callback: Arc, - ) -> Result<(), Error> { + fn register_remove_callback(self: Arc, callback: RemoveCallback) -> Result<(), Error> { self.inner_store.register_remove_callback(callback) } } diff --git a/nativelink-store/src/dedup_store.rs b/nativelink-store/src/dedup_store.rs index bde8ed7c5..d37850f8d 100644 --- a/nativelink-store/src/dedup_store.rs +++ b/nativelink-store/src/dedup_store.rs @@ -27,7 +27,7 @@ use nativelink_util::common::DigestInfo; use nativelink_util::fastcdc::FastCDC; use nativelink_util::health_utils::{HealthStatusIndicator, default_health_status_indicator}; use nativelink_util::store_trait::{ - RemoveItemCallback, Store, StoreDriver, StoreKey, StoreLike, UploadSizeInfo, + RemoveCallback, Store, StoreDriver, StoreKey, StoreLike, UploadSizeInfo, }; use serde::{Deserialize, Serialize}; use tokio_util::codec::FramedRead; @@ -396,10 +396,7 @@ impl StoreDriver for DedupStore { self } - fn register_remove_callback( - self: Arc, - callback: Arc, - ) -> Result<(), Error> { + fn register_remove_callback(self: Arc, callback: RemoveCallback) -> Result<(), Error> { self.index_store .register_remove_callback(callback.clone())?; self.content_store.register_remove_callback(callback)?; diff --git a/nativelink-store/src/default_store_factory.rs b/nativelink-store/src/default_store_factory.rs index 01ae9d064..ec92a5069 100644 --- a/nativelink-store/src/default_store_factory.rs +++ b/nativelink-store/src/default_store_factory.rs @@ -18,7 +18,9 @@ use std::time::SystemTime; use futures::stream::FuturesOrdered; use futures::{Future, TryStreamExt}; -use nativelink_config::stores::{ExperimentalCloudObjectSpec, RedisMode, StoreSpec}; +use nativelink_config::stores::{ + CompressionAlgorithm, ExperimentalCloudObjectSpec, RedisMode, StoreSpec, +}; use nativelink_error::Error; use nativelink_util::health_utils::HealthRegistryBuilder; use nativelink_util::store_trait::{Store, StoreDriver}; @@ -47,6 +49,7 @@ use crate::shard_store::ShardStore; use crate::size_partitioning_store::SizePartitioningStore; use crate::store_manager::StoreManager; use crate::verify_store::VerifyStore; +use crate::zstd_store::ZstdStore; type FutureMaybeStore<'a> = Box> + Send + 'a>; @@ -93,10 +96,13 @@ pub fn store_factory<'a>( spec, store_factory(&spec.backend, store_manager, None).await?, ), - StoreSpec::Compression(spec) => CompressionStore::new( - &spec.clone(), - store_factory(&spec.backend, store_manager, None).await?, - )?, + StoreSpec::Compression(spec) => { + let inner = store_factory(&spec.backend, store_manager, None).await?; + match &spec.compression_algorithm { + CompressionAlgorithm::Lz4(_) => CompressionStore::new(spec, inner)?, + CompressionAlgorithm::Zstd(config) => ZstdStore::new(config, inner)?, + } + } StoreSpec::Dedup(spec) => DedupStore::new( spec, store_factory(&spec.index_store, store_manager, None).await?, diff --git a/nativelink-store/src/existence_cache_store.rs b/nativelink-store/src/existence_cache_store.rs index a26b5796e..ffd2625f1 100644 --- a/nativelink-store/src/existence_cache_store.rs +++ b/nativelink-store/src/existence_cache_store.rs @@ -29,7 +29,7 @@ use nativelink_util::evicting_map::{EvictingMap, LenEntry}; use nativelink_util::health_utils::{HealthStatus, HealthStatusIndicator}; use nativelink_util::instant_wrapper::InstantWrapper; use nativelink_util::store_trait::{ - RemoveItemCallback, Store, StoreDriver, StoreKey, StoreLike, UploadSizeInfo, + RemoveCallback, RemoveItemCallback, Store, StoreDriver, StoreKey, StoreLike, UploadSizeInfo, }; use parking_lot::Mutex; use tracing::{debug, info, trace}; @@ -312,10 +312,7 @@ impl StoreDriver for ExistenceCacheStore { self } - fn register_remove_callback( - self: Arc, - callback: Arc, - ) -> Result<(), Error> { + fn register_remove_callback(self: Arc, callback: RemoveCallback) -> Result<(), Error> { self.inner_store.register_remove_callback(callback) } } diff --git a/nativelink-store/src/fast_slow_store.rs b/nativelink-store/src/fast_slow_store.rs index d7ae49a05..a951b60da 100644 --- a/nativelink-store/src/fast_slow_store.rs +++ b/nativelink-store/src/fast_slow_store.rs @@ -34,8 +34,8 @@ use nativelink_util::buf_channel::{ use nativelink_util::fs; use nativelink_util::health_utils::{HealthStatusIndicator, default_health_status_indicator}; use nativelink_util::store_trait::{ - RemoveItemCallback, Store, StoreDriver, StoreKey, StoreLike, StoreOptimizations, - UploadSizeInfo, slow_update_store_with_file, + RemoveCallback, Store, StoreDriver, StoreKey, StoreLike, StoreOptimizations, UploadSizeInfo, + slow_update_store_with_file, }; use parking_lot::Mutex; use tokio::sync::OnceCell; @@ -274,7 +274,7 @@ impl FastSlowStore { if let StoreKey::Digest(d) = key.borrow() { err.with_context(ErrorContext::MissingDigest { hash: d.packed_hash().to_string(), - size: d.size_bytes() as i64, + size: d.size_bytes().try_into().unwrap_or(i64::MAX), }) } else { err @@ -928,10 +928,7 @@ impl StoreDriver for FastSlowStore { self } - fn register_remove_callback( - self: Arc, - callback: Arc, - ) -> Result<(), Error> { + fn register_remove_callback(self: Arc, callback: RemoveCallback) -> Result<(), Error> { self.fast_store.register_remove_callback(callback.clone())?; self.slow_store.register_remove_callback(callback)?; Ok(()) diff --git a/nativelink-store/src/filesystem_store.rs b/nativelink-store/src/filesystem_store.rs index 03cf08fdc..8654254ec 100644 --- a/nativelink-store/src/filesystem_store.rs +++ b/nativelink-store/src/filesystem_store.rs @@ -43,8 +43,10 @@ use nativelink_util::fs::FileSlot; use nativelink_util::health_utils::{HealthRegistryBuilder, HealthStatus, HealthStatusIndicator}; #[cfg(unix)] use nativelink_util::spawn_blocking; +#[cfg(unix)] +use nativelink_util::store_trait::RemoveItemCallback; use nativelink_util::store_trait::{ - RemoveItemCallback, StoreDriver, StoreKey, StoreKeyBorrow, StoreOptimizations, UploadSizeInfo, + RemoveCallback, StoreDriver, StoreKey, StoreKeyBorrow, StoreOptimizations, UploadSizeInfo, }; use tokio::io::{AsyncReadExt, AsyncWriteExt, Take}; use tokio::sync::Semaphore; @@ -52,7 +54,7 @@ use tokio::time::timeout; use tokio_stream::wrappers::ReadDirStream; use tracing::{debug, error, info, trace, warn}; -use crate::callback_utils::RemoveItemCallbackHolder; +use crate::callback_utils::RemoveCallbackHolder; use crate::cas_utils::is_zero_digest; // Default size to allocate memory of the buffer when reading files. @@ -67,7 +69,9 @@ pub const DIGEST_FOLDER: &str = "d"; /// **executable** (0o555) variants of CAS blobs (see /// [`FilesystemStore::get_executable_hardlink_source`]). It is a sibling of /// `content_path` rather than a child so the normal content/temp scan and prune -/// logic never touches it. Cleared on startup; entries are regenerable. +/// logic never touches it. Cleared on writable startup; entries are +/// regenerable. If the wipe is blocked by a read-only filesystem, executable +/// variants are disabled so surviving files are never trusted. #[cfg(unix)] const EXECUTABLE_DIR_SUFFIX: &str = ".exec"; @@ -376,6 +380,272 @@ pub fn make_temp_key(key: &StoreKey) -> StoreKey<'static> { StoreKey::Digest(make_temp_digest(key.borrow().into_digest())) } +/// Group-commit flush coalescer (macOS only). +/// +/// On macOS, `File::sync_all` issues `fcntl(F_FULLFSYNC)` — a full +/// device-cache flush — once per call. The flush is serialized at the +/// device, costs multiple milliseconds, and dominates uploads of many +/// small blobs (a 4,400-tiny-file action input tree spends ~12s of its +/// ~14.5s materialization in these flushes). Linux does not have this +/// problem because concurrent `fsync` calls coalesce inside the +/// filesystem journal's group commit. +/// +/// This applies the same group-commit idea in userspace, exploiting an +/// asymmetry in `F_FULLFSYNC`: writing a file's pages to the storage +/// device is per-file work (plain `fsync(2)`, cheap), while the expensive +/// device-cache drain is device-wide by nature. Each writer first pushes +/// its own data to the device with `fsync(2)`, then joins the current +/// commit round; a single long-lived flusher task per DEVICE issues one +/// `F_FULLFSYNC` (on a dedicated sentinel file) per round, covering every +/// previously-`fsync`ed blob at once. Rounds self-batch exactly like a +/// journal: while one flush is running, later writers accumulate into +/// the next round. +/// +/// The coalescer (and its flusher task) is per store instance. Sharing +/// one coalescer across all stores on a device would amortize further — +/// the flush is device-wide — but a process-global flusher task cannot +/// safely outlive the tokio runtime that spawned it (multiple runtimes +/// coexist in one process, e.g. one per test); doing this correctly +/// needs a runtime-agnostic flusher (a dedicated OS thread) and is left +/// as a follow-up. +/// +/// Durability is identical to per-file `F_FULLFSYNC`: a writer only +/// proceeds (and only renames its blob into the content directory) after +/// a device-cache flush that started after its own `fsync(2)` completed. +#[cfg(target_os = "macos")] +#[derive(Debug)] +struct FlushCoalescer { + /// The round currently accepting waiters. Swapped out by the flusher + /// right before it flushes, so writers whose `fsync(2)` finished + /// after the flush began land in the next round. + current_round: parking_lot::Mutex>, + /// Wakes the flusher task. Writers notify after subscribing to their + /// round, so a wakeup can never be observed before its waiter. + wake: Arc, + /// Dedicated file the `F_FULLFSYNC` is issued on. A SIBLING of the + /// store's content path (like the `.exec` variant directory), so + /// neither the startup scan nor `move_old_cache`'s legacy root sweep + /// ever sees it. + sentinel: Arc, + /// Number of device-wide barriers issued. Kept separately from the + /// per-file `fsync(2)` count so tests can pin the coalescing invariant. + full_flush_count: Arc, +} + +#[cfg(target_os = "macos")] +#[derive(Debug)] +struct FlushRound { + /// Broadcasts the flush outcome to every writer in the round. + result_tx: tokio::sync::watch::Sender>>, +} + +#[cfg(target_os = "macos")] +impl FlushRound { + fn new() -> Arc { + let (result_tx, _) = tokio::sync::watch::channel(None); + Arc::new(Self { result_tx }) + } +} + +/// Guarantees a [`FlushRound`]'s waiters always receive a result: if the +/// flusher task is cancelled at an await point (runtime shutdown), waiters +/// must get an error rather than pend forever — their own `Arc` +/// keeps the channel alive, so a dropped-sender wakeup can never happen. +#[cfg(target_os = "macos")] +struct SendOnDrop(Option>); + +#[cfg(target_os = "macos")] +impl SendOnDrop { + fn finish(mut self, result: Result<(), Error>) { + if let Some(round) = self.0.take() { + // Ignore send errors: every waiter may have been cancelled. + drop(round.result_tx.send(Some(result))); + } + } +} + +#[cfg(target_os = "macos")] +impl Drop for SendOnDrop { + fn drop(&mut self) { + if let Some(round) = self.0.take() { + drop(round.result_tx.send(Some(Err(make_err!( + Code::Internal, + "Flush coalescer round task cancelled before completing" + ))))); + } + } +} + +#[cfg(target_os = "macos")] +impl Drop for FlushCoalescer { + fn drop(&mut self) { + // Wake the flusher so it observes the dead Weak and exits instead + // of parking forever. + self.wake.notify_one(); + } +} + +#[cfg(target_os = "macos")] +impl FlushCoalescer { + /// Creates the coalescer for a store and spawns its flusher task on + /// the current runtime (the same runtime the store's uploads run on). + async fn for_content_path(content_path: &str) -> Result, Error> { + // Sibling of `content_path` (never inside it): the startup scan + // and `move_old_cache` sweep everything under the content root. + let sentinel_path = format!("{content_path}.flush_sentinel"); + let sentinel = spawn_blocking!("filesystem_store_flush_sentinel_open", move || { + std::fs::OpenOptions::new() + .write(true) + .create(true) + .truncate(false) + .open(&sentinel_path) + .map_err(|e| { + make_err!( + Code::Internal, + "Failed to create flush sentinel {sentinel_path}: {e:?}" + ) + }) + }) + .await + .err_tip(|| "Failed to join flush sentinel open task")??; + + let coalescer = Arc::new(Self { + current_round: parking_lot::Mutex::new(FlushRound::new()), + wake: Arc::new(tokio::sync::Notify::new()), + sentinel: Arc::new(sentinel), + full_flush_count: Arc::new(AtomicU64::new(0)), + }); + + let weak = Arc::downgrade(&coalescer); + let wake = coalescer.wake.clone(); + background_spawn!("filesystem_store_flush_coalescer", async move { + loop { + wake.notified().await; + let Some(coalescer) = weak.upgrade() else { + return; + }; + coalescer.flush_one_round().await; + // Drop the strong ref before parking so the store can be + // torn down while the flusher is idle. + } + }); + Ok(coalescer) + } + + /// Waits until a device-cache flush that started after this call has + /// completed. Callers must have already `fsync(2)`ed their own file. + async fn commit(&self) -> Result<(), Error> { + let round = self.current_round.lock().clone(); + let mut result_rx = round.result_tx.subscribe(); + // Notify strictly after subscribing: the flusher skips rounds + // with no receivers, so this ordering makes lost wakeups + // impossible (a permit is stored even while the flusher is busy). + self.wake.notify_one(); + let result_ref = result_rx + .wait_for(Option::is_some) + .await + .map_err(|_| make_err!(Code::Internal, "Flush coalescer round dropped"))?; + result_ref + .clone() + .unwrap_or_else(|| Err(make_err!(Code::Internal, "Flush round result missing"))) + } + + async fn flush_one_round(&self) { + let round = self.current_round.lock().clone(); + if round.result_tx.receiver_count() == 0 { + // Spurious wakeup (e.g. a permit left over from a round that + // a previous iteration already served). + return; + } + let send_guard = SendOnDrop(Some(round.clone())); + + // Brief accumulation window, only when this round actually has + // multiple waiters: lets a burst pack more writers into the round + // before it closes, trading ~3ms of publish latency (about the + // cost of one device flush) for fewer device flushes. A solo + // writer on an idle store skips it and pays only its own flush. + if round.result_tx.receiver_count() > 1 { + tokio::time::sleep(Duration::from_millis(3)).await; + } + // Close the round: writers arriving from here on cannot assume + // this flush covers them, so they must get a fresh round. + { + let mut current = self.current_round.lock(); + if Arc::ptr_eq(&*current, &round) { + *current = FlushRound::new(); + } + } + let sentinel = self.sentinel.clone(); + let full_flush_count = self.full_flush_count.clone(); + let flush_result = spawn_blocking!("filesystem_store_full_flush", move || { + use std::os::fd::AsRawFd; + use std::os::unix::fs::FileExt; + // Keep the sentinel dirty so the flush is never a no-op. + // Best-effort: a failed write (e.g. ENOSPC on a full CoW + // volume) must not fail the whole round — the fcntl below is + // what provides the device-cache drain. + if let Err(err) = sentinel.write_at(b"f", 0) { + warn!(?err, "Flush sentinel write failed; issuing flush anyway"); + } + // Count the actual blocking operation, rather than merely counting + // rounds scheduled by the async task. + full_flush_count.fetch_add(1, Ordering::Relaxed); + loop { + if unsafe { libc::fcntl(sentinel.as_raw_fd(), libc::F_FULLFSYNC) } != -1 { + return Ok(()); + } + let err = std::io::Error::last_os_error(); + // std's sync_all retries EINTR (cvt_r); match it. + if err.kind() != std::io::ErrorKind::Interrupted { + return Err(Error::from(err).append("F_FULLFSYNC failed in flush coalescer")); + } + } + }) + .await + .unwrap_or_else(|e| { + Err(Error::from_std_err(Code::Internal, &e).append("Flush coalescer task failed")) + }); + send_guard.finish(flush_result); + } +} + +#[cfg(unix)] +async fn prepare_executable_dir(content_path: &str) -> Result { + fn is_non_writable_error(err: &std::io::Error) -> bool { + matches!( + err.kind(), + std::io::ErrorKind::PermissionDenied | std::io::ErrorKind::ReadOnlyFilesystem + ) + } + + let executable_dir = format!("{content_path}{EXECUTABLE_DIR_SUFFIX}"); + let executable_digest_dir = format!("{executable_dir}/{DIGEST_FOLDER}"); + spawn_blocking!("filesystem_store_prepare_executable_dir", move || { + match std::fs::remove_dir_all(&executable_dir) { + Ok(()) => {} + Err(err) if err.kind() == std::io::ErrorKind::NotFound => {} + // A read-only cache must remain available for ordinary reads. Do + // not reuse the directory, though: a surviving executable variant + // may be stale or torn after an unclean shutdown. + Err(err) if is_non_writable_error(&err) => return Ok(false), + Err(err) => { + return Err(Error::from(err) + .append(format!("Failed to clear executable dir {executable_dir}"))); + } + } + + match std::fs::create_dir_all(&executable_digest_dir) { + Ok(()) => Ok(true), + Err(err) if is_non_writable_error(&err) => Ok(false), + Err(err) => Err(Error::from(err).append(format!( + "Failed to create executable dir {executable_digest_dir}" + ))), + } + }) + .await + .err_tip(|| "Failed to join executable-dir preparation task")? +} + impl LenEntry for FileEntryImpl { #[inline] fn len(&self) -> u64 { @@ -476,7 +746,7 @@ pub fn key_from_file(file_name: &str, file_type: FileType) -> Result = - EvictingMap, Arc, SystemTime, RemoveItemCallbackHolder>; + EvictingMap, Arc, SystemTime, RemoveCallbackHolder>; async fn add_files_to_cache( evicting_map: &FsEvictingMap<'_, Fe>, @@ -840,16 +1110,31 @@ pub struct FilesystemStore { block_size: u64, #[metric(help = "Size of the configured read buffer size")] read_buffer_size: usize, + /// See [`FilesystemSpec::evict_page_cache`]. When false (the default) the + /// per-blob `posix_fadvise(DONTNEED)` calls are skipped. + evict_page_cache: bool, weak_self: Weak, rename_fn: fn(&OsStr, &OsStr) -> Result<(), std::io::Error>, /// Limits concurrent write operations to prevent disk I/O saturation. write_semaphore: Option, + /// See [`FlushCoalescer`]: amortizes the per-blob `F_FULLFSYNC` cost + /// across concurrent uploads without weakening durability. `None` when + /// the sentinel could not be created (e.g. read-only content volume); + /// uploads then fall back to per-file `sync_all`, matching the old + /// behavior (and such stores fail at write time anyway). + #[cfg(target_os = "macos")] + flush_coalescer: Option>, /// Per-digest single-flight locks guarding creation of the executable /// variant in `{content_path}.exec`, so each variant's writable fd is /// opened exactly once. The outer lock is sync and only ever held to /// get/insert/remove the per-digest async lock — never across I/O. #[cfg(unix)] executable_locks: std::sync::Mutex>>>, + /// False when the startup wipe could not safely remove old executable + /// variants from a read-only filesystem. Ordinary CAS reads remain + /// available, but executable hardlink sources must not reuse stale files. + #[cfg(unix)] + executable_variants_enabled: bool, } impl FilesystemStore { @@ -883,23 +1168,33 @@ impl FilesystemStore { // Executable-variant directory: a sibling of `content_path` holding // per-digest 0o555 copies used as hardlink sources for executable - // inputs (see `get_executable_hardlink_source`). Cleared on startup — - // the variants are regenerable and we never want a stale one to leak - // across runs. Unix-only: the executable bit (and the ETXTBSY race it - // guards against) does not apply on Windows. + // inputs (see `get_executable_hardlink_source`). Cleared on writable + // startup — the variants are regenerable and we never want a stale one + // to leak across runs. If a read-only filesystem prevents the wipe, + // ordinary CAS reads remain enabled but executable variants do not. + // Unix-only: the executable bit (and the ETXTBSY race it guards + // against) does not apply on Windows. #[cfg(unix)] - { - let executable_dir = format!("{}{EXECUTABLE_DIR_SUFFIX}", spec.content_path); - drop(fs::remove_dir_all(&executable_dir).await); - fs::create_dir_all(format!("{executable_dir}/{DIGEST_FOLDER}")) - .await - .err_tip(|| format!("Failed to create executable dir {executable_dir}"))?; - evicting_map.add_remove_callback(RemoveItemCallbackHolder::new(Arc::new( - ExecutableVariantRemover { - content_path: spec.content_path.clone(), - }, - ))); - } + let executable_variants_enabled = { + let enabled = prepare_executable_dir(&spec.content_path).await?; + if !enabled { + warn!( + executable_dir = %format!("{}{EXECUTABLE_DIR_SUFFIX}", spec.content_path), + "Executable directory is not writable; serving CAS reads with executable variants disabled" + ); + } + if enabled { + // Only register cleanup when executable variants are enabled. + // Otherwise surviving variants are deliberately quarantined, + // and repeated deletion warnings would obscure the fallback. + evicting_map.add_remove_callback(RemoveCallbackHolder::new(Arc::new( + ExecutableVariantRemover { + content_path: spec.content_path.clone(), + }, + ))); + } + enabled + }; let shared_context = Arc::new(SharedContext { active_drop_spawns: AtomicU64::new(0), @@ -932,16 +1227,35 @@ impl FilesystemStore { } else { None }; + // Never make construction fail over the durability optimization: + // a read-only content volume must still serve reads, exactly as + // it did before the coalescer existed. + #[cfg(target_os = "macos")] + let flush_coalescer = FlushCoalescer::for_content_path(&shared_context.content_path) + .await + .inspect_err(|err| { + warn!( + ?err, + "Failed to create flush coalescer; falling back to per-file sync_all" + ); + }) + .ok(); + Ok(Arc::new_cyclic(|weak_self| Self { shared_context, evicting_map, block_size, read_buffer_size, + evict_page_cache: spec.evict_page_cache, weak_self: weak_self.clone(), rename_fn, write_semaphore, + #[cfg(target_os = "macos")] + flush_coalescer, #[cfg(unix)] executable_locks: std::sync::Mutex::new(HashMap::new()), + #[cfg(unix)] + executable_variants_enabled, })) } @@ -980,6 +1294,12 @@ impl FilesystemStore { &self, digest: &DigestInfo, ) -> Result { + if !self.executable_variants_enabled { + return Err(make_err!( + Code::FailedPrecondition, + "Executable hardlink sources are disabled because the startup wipe could not safely clear the read-only executable directory" + )); + } let variant_path = self.executable_variant_path(digest); // Fast path: the variant already exists, so the caller can hardlink it @@ -1097,10 +1417,33 @@ impl FilesystemStore { "executable-variant chmod 0o555 failed: {e:?}" ) })?; - // Reopen read-only purely to fsync the bytes durable before publish. + // Belt-and-suspenders flush before publish. The `.exec` + // directory is cleared before executable variants are enabled, + // so durability is not needed across process restarts. The + // flush still ensures a published variant is complete for the + // current process. Non-macOS keeps the prior `sync_all` + // behavior; macOS uses plain `fsync(2)` (data handed to the + // device, no multi-ms `F_FULLFSYNC` device-cache drain), which + // covers kernel panics — the next writable startup wipe covers + // power loss. let f = std::fs::File::open(&temp_owned) .map_err(|e| make_err!(Code::Internal, "executable-variant reopen: {e:?}"))?; - f.sync_all() + #[cfg(target_os = "macos")] + let flush_result = { + use std::os::fd::AsRawFd; + loop { + if unsafe { libc::fsync(f.as_raw_fd()) } == 0 { + break Ok(()); + } + let err = std::io::Error::last_os_error(); + if err.kind() != std::io::ErrorKind::Interrupted { + break Err(err); + } + } + }; + #[cfg(not(target_os = "macos"))] + let flush_result = f.sync_all(); + flush_result .map_err(|e| make_err!(Code::Internal, "executable-variant fsync: {e:?}"))?; drop(f); rename_fn(temp_owned.as_os_str(), variant_owned.as_os_str()).map_err(|e| { @@ -1173,15 +1516,15 @@ impl FilesystemStore { .flush() .await .err_tip(|| "Failed to flush in filesystem store")?; - temp_file - .as_ref() - .sync_all() + self.flush_durably(&temp_file) .await - .err_tip(|| "Failed to sync_data in filesystem store")?; + .err_tip(|| "Failed to sync in filesystem store")?; drop(permit); - temp_file.advise_dontneed(); + if self.evict_page_cache { + temp_file.advise_dontneed(); + } trace!(?temp_file, "Dropping file to update_file"); drop(temp_file); @@ -1300,6 +1643,61 @@ impl FilesystemStore { .err_tip(|| "Failed to create spawn in filesystem store update_file")? } + /// Makes `file`'s contents durable with `sync_all` semantics. + /// + /// On macOS the naive equivalent (`fcntl(F_FULLFSYNC)` per file) is a + /// full device-cache flush that serializes at the device and dominates + /// many-small-blob uploads, so the flush is split into the cheap + /// per-file part (`fsync(2)`: push this file's pages to the device) + /// and a device-wide cache drain shared with every concurrent upload + /// via [`FlushCoalescer`]. Other platforms get their journal's group + /// commit for free and keep plain `sync_all`. + async fn flush_durably(&self, file: &FileSlot) -> Result<(), Error> { + #[cfg(target_os = "macos")] + { + use std::os::fd::AsRawFd; + let Some(flush_coalescer) = &self.flush_coalescer else { + return file.as_ref().sync_all().await.map_err(Into::into); + }; + // Dup the handle so the blocking task owns its own fd: if + // this future is cancelled mid-await, the temp file's fd can + // close and be reused, and an already-started blocking task + // must never fsync an unrelated descriptor. + let file_dup = file + .as_ref() + .try_clone() + .await + .err_tip(|| "Failed to dup fd for fsync in filesystem store")? + .into_std() + .await; + // Deliberately NOT via `fs::call_with_permit`: the caller's + // `FileSlot` already holds an open-file permit, so requiring a + // second one here deadlocks when the semaphore is drained (the + // exact scenario the #2051 regression test pins at one + // permit). Concurrency is still bounded: every in-flight fsync + // belongs to an upload holding a `FileSlot` permit. + spawn_blocking!("filesystem_store_fsync", move || { + loop { + if unsafe { libc::fsync(file_dup.as_raw_fd()) } == 0 { + return Ok(()); + } + let err = std::io::Error::last_os_error(); + // std's sync_all retries EINTR (cvt_r); match it. + if err.kind() != std::io::ErrorKind::Interrupted { + return Err(Error::from(err).append("fsync failed in filesystem store")); + } + } + }) + .await + .err_tip(|| "Failed to join fsync task in filesystem store")??; + flush_coalescer.commit().await + } + #[cfg(not(target_os = "macos"))] + { + file.as_ref().sync_all().await.map_err(Into::into) + } + } + pub fn get_eviction_snapshot(&self) -> EvictionSnapshot { self.evicting_map.get_snapshot() } @@ -1309,6 +1707,16 @@ impl FilesystemStore { self.evicting_map.clone() } + /// Returns the number of device-wide durability barriers issued by this + /// store's macOS flush coalescer. Exposed so the integration test can pin + /// the batching behavior rather than only checking data correctness. + #[cfg(target_os = "macos")] + pub fn full_flush_count_for_test(&self) -> Option { + self.flush_coalescer + .as_ref() + .map(|coalescer| coalescer.full_flush_count.load(Ordering::Relaxed)) + } + // Separated out so tests can use this pub async fn make_temp_file( &self, @@ -1438,15 +1846,15 @@ impl StoreDriver for FilesystemStore { .flush() .await .err_tip(|| "Failed to flush in filesystem store update_oneshot")?; - temp_file - .as_ref() - .sync_all() + self.flush_durably(&temp_file) .await - .err_tip(|| "Failed to sync_data in filesystem store update_oneshot")?; + .err_tip(|| "Failed to sync in filesystem store update_oneshot")?; drop(_permit); - temp_file.advise_dontneed(); + if self.evict_page_cache { + temp_file.advise_dontneed(); + } drop(temp_file); *entry.data_size_mut() = data.len() as u64; @@ -1485,7 +1893,9 @@ impl StoreDriver for FilesystemStore { // We are done with the file, if we hold a reference to the file here, it could // result in a deadlock if `emplace_file()` also needs file descriptors. trace!(?file, "Dropping file to to update_with_whole_file"); - file.advise_dontneed(); + if self.evict_page_cache { + file.advise_dontneed(); + } drop(file); self.emplace_file(key.into_owned(), Arc::new(entry)) .await @@ -1549,7 +1959,9 @@ impl StoreDriver for FilesystemStore { .await .err_tip(|| "Failed to send chunk in filesystem store get_part")?; } - temp_file.get_ref().advise_dontneed(); + if self.evict_page_cache { + temp_file.get_ref().advise_dontneed(); + } writer .send_eof() .err_tip(|| "Filed to send EOF in filesystem store get_part")?; @@ -1573,12 +1985,9 @@ impl StoreDriver for FilesystemStore { registry.register_indicator(self); } - fn register_remove_callback( - self: Arc, - callback: Arc, - ) -> Result<(), Error> { + fn register_remove_callback(self: Arc, callback: RemoveCallback) -> Result<(), Error> { self.evicting_map - .add_remove_callback(RemoveItemCallbackHolder::new(callback)); + .add_remove_callback(RemoveCallbackHolder::new(callback)); Ok(()) } } diff --git a/nativelink-store/src/gcs_client/client.rs b/nativelink-store/src/gcs_client/client.rs index 5e812ea7a..d8e374862 100644 --- a/nativelink-store/src/gcs_client/client.rs +++ b/nativelink-store/src/gcs_client/client.rs @@ -216,7 +216,7 @@ impl GcsClient { /// Convert GCS object to our internal representation fn convert_to_gcs_object(&self, obj: Object) -> GcsObject { let update_time = obj.updated.map(|dt| Timestamp { - seconds: dt.unix_timestamp(), + seconds: dt.unix_timestamp() as u64, nanos: 0, }); diff --git a/nativelink-store/src/gcs_client/mocks.rs b/nativelink-store/src/gcs_client/mocks.rs index ee4662691..f5ba13b55 100644 --- a/nativelink-store/src/gcs_client/mocks.rs +++ b/nativelink-store/src/gcs_client/mocks.rs @@ -151,12 +151,12 @@ impl MockGcsOperations { let now = SystemTime::now() .duration_since(UNIX_EPOCH) .unwrap_or_default() - .as_secs() as i64; + .as_secs(); let metadata = GcsObject { name: path.path.clone(), bucket: path.bucket.clone(), - size: content.len() as i64, + size: content.len().try_into().unwrap_or(i64::MAX), content_type: DEFAULT_CONTENT_TYPE.to_string(), update_time: Some(Timestamp { seconds: now, @@ -244,14 +244,14 @@ impl MockGcsOperations { &self, path: &ObjectPath, content: Vec, - timestamp: i64, + timestamp: u64, ) { let object_key = self.get_object_key(path); let metadata = GcsObject { name: path.path.clone(), bucket: path.bucket.clone(), - size: content.len() as i64, + size: content.len().try_into().unwrap_or(i64::MAX), content_type: DEFAULT_CONTENT_TYPE.to_string(), update_time: Some(Timestamp { seconds: timestamp, @@ -264,11 +264,11 @@ impl MockGcsOperations { } /// Get the current timestamp - fn get_current_timestamp(&self) -> i64 { + fn get_current_timestamp(&self) -> u64 { SystemTime::now() .duration_since(UNIX_EPOCH) .unwrap_or_default() - .as_secs() as i64 + .as_secs() } } @@ -450,7 +450,7 @@ impl GcsOperations for MockGcsOperations { // Update metadata if this is the final chunk if total_size.is_some_and(|size| size == end_offset) { - mock_object.metadata.size = mock_object.content.len() as i64; + mock_object.metadata.size = mock_object.content.len().try_into().unwrap_or(i64::MAX); mock_object.metadata.update_time = Some(Timestamp { seconds: self.get_current_timestamp(), nanos: 0, diff --git a/nativelink-store/src/gcs_client/types.rs b/nativelink-store/src/gcs_client/types.rs index ab667c443..94350f4fc 100644 --- a/nativelink-store/src/gcs_client/types.rs +++ b/nativelink-store/src/gcs_client/types.rs @@ -48,8 +48,8 @@ pub struct GcsObject { #[derive(Debug, Clone, Copy)] pub struct Timestamp { - pub seconds: i64, - pub nanos: i32, + pub seconds: u64, + pub nanos: u32, } #[derive(Clone, Debug)] diff --git a/nativelink-store/src/gcs_store.rs b/nativelink-store/src/gcs_store.rs index 247ced282..dd9731a5c 100644 --- a/nativelink-store/src/gcs_store.rs +++ b/nativelink-store/src/gcs_store.rs @@ -30,7 +30,7 @@ use nativelink_util::health_utils::{HealthRegistryBuilder, HealthStatus, HealthS use nativelink_util::instant_wrapper::InstantWrapper; use nativelink_util::retry::{Retrier, RetryResult}; use nativelink_util::store_trait::{ - RemoveItemCallback, StoreDriver, StoreKey, StoreOptimizations, UploadSizeInfo, + RemoveCallback, StoreDriver, StoreKey, StoreOptimizations, UploadSizeInfo, }; use rand::Rng; use tokio::time::{sleep, timeout}; @@ -53,7 +53,7 @@ pub struct GcsStore { key_prefix: String, retrier: Retrier, #[metric(help = "The number of seconds to consider an object expired")] - consider_expired_after_s: i64, + consider_expired_after_s: u64, #[metric(help = "The number of bytes to buffer for retrying requests")] max_retry_buffer_size: usize, #[metric(help = "The size of chunks for resumable uploads")] @@ -137,7 +137,7 @@ where jitter_fn, spec.common.retry.clone(), ), - consider_expired_after_s: i64::from(spec.common.consider_expired_after_s), + consider_expired_after_s: u64::from(spec.common.consider_expired_after_s), max_retry_buffer_size, max_chunk_size, max_concurrent_uploads: max_connections, @@ -162,7 +162,7 @@ where if consider_expired_after_s != 0 && let Some(update_time) = &metadata.update_time { - let now_s = now_fn().unix_timestamp() as i64; + let now_s = now_fn().unix_timestamp(); if update_time.seconds + consider_expired_after_s <= now_s { return Some((RetryResult::Ok(None), object_path)); } @@ -429,15 +429,24 @@ where .await { Ok(stream) => stream, - Err(e) if e.code == Code::NotFound => { - return Some((RetryResult::Err(e), (offset, writer))); - } + // NotFound is intentionally not special-cased here: + // the retrier doesn't retry NotFound by default, but + // emitting `Retry` lets `retry.retry_on_errors` opt + // reads into retrying read-after-write races where + // an object is still finalizing or being repopulated. Err(e) => return Some((RetryResult::Retry(e), (offset, writer))), }; while let Some(next_chunk) = stream.next().await { match next_chunk { Ok(bytes) => { + // HTTP/2 peers may emit zero-length DATA frames + // (e.g. a bare END_STREAM frame), which surface + // here as empty chunks. `send()` rejects empty + // buffers by contract, so skip them. + if bytes.is_empty() { + continue; + } offset += bytes.len() as u64; if let Err(err) = writer.send(bytes).await { return Some((RetryResult::Err(err), (offset, writer))); @@ -473,10 +482,7 @@ where registry.register_indicator(self); } - fn register_remove_callback( - self: Arc, - _callback: Arc, - ) -> Result<(), Error> { + fn register_remove_callback(self: Arc, _callback: RemoveCallback) -> Result<(), Error> { // As we're backed by GCS, this store doesn't actually drop stuff // so we can actually just ignore this Ok(()) diff --git a/nativelink-store/src/grpc_store.rs b/nativelink-store/src/grpc_store.rs index 79d866179..a40afdb85 100644 --- a/nativelink-store/src/grpc_store.rs +++ b/nativelink-store/src/grpc_store.rs @@ -13,15 +13,17 @@ // limitations under the License. use core::pin::Pin; +use core::sync::atomic::{AtomicU64, Ordering}; use core::time::Duration; use std::borrow::Cow; -use std::sync::Arc; +use std::collections::{HashMap, VecDeque}; +use std::sync::{Arc, Weak}; use async_trait::async_trait; -use bytes::BytesMut; +use bytes::{Bytes, BytesMut}; use futures::stream::{FuturesUnordered, unfold}; use futures::{Future, Stream, StreamExt, TryFutureExt, TryStreamExt, future}; -use nativelink_config::stores::GrpcSpec; +use nativelink_config::stores::{GrpcReadBatchingConfig, GrpcSpec}; use nativelink_error::{Error, ResultExt, error_if, make_err}; use nativelink_metric::MetricsComponent; use nativelink_proto::build::bazel::remote::execution::v2::action_cache_client::ActionCacheClient; @@ -30,14 +32,16 @@ use nativelink_proto::build::bazel::remote::execution::v2::{ ActionResult, BatchReadBlobsRequest, BatchReadBlobsResponse, BatchUpdateBlobsRequest, BatchUpdateBlobsResponse, FindMissingBlobsRequest, FindMissingBlobsResponse, GetActionResultRequest, GetTreeRequest, GetTreeResponse, SpliceBlobRequest, SpliceBlobResponse, - SplitBlobRequest, SplitBlobResponse, UpdateActionResultRequest, + SplitBlobRequest, SplitBlobResponse, UpdateActionResultRequest, compressor, }; use nativelink_proto::google::bytestream::byte_stream_client::ByteStreamClient; use nativelink_proto::google::bytestream::{ QueryWriteStatusRequest, QueryWriteStatusResponse, ReadRequest, ReadResponse, WriteRequest, WriteResponse, }; -use nativelink_util::buf_channel::{DropCloserReadHalf, DropCloserWriteHalf}; +use nativelink_util::buf_channel::{ + DropCloserReadHalf, DropCloserWriteHalf, make_buf_channel_pair, +}; use nativelink_util::common::DigestInfo; use nativelink_util::connection_manager::ConnectionManager; use nativelink_util::digest_hasher::{DigestHasherFunc, default_digest_hasher_func}; @@ -47,18 +51,22 @@ use nativelink_util::proto_stream_utils::{ }; use nativelink_util::resource_info::ResourceInfo; use nativelink_util::retry::{Retrier, RetryResult}; -use nativelink_util::store_trait::{RemoveItemCallback, StoreDriver, StoreKey, UploadSizeInfo}; +use nativelink_util::store_trait::{RemoveCallback, StoreDriver, StoreKey, UploadSizeInfo}; use nativelink_util::telemetry::ClientHeaders; -use nativelink_util::{default_health_status_indicator, tls_utils}; +use nativelink_util::wire_compression::{ + stream_decode_compressed_upload, stream_encode_compressed_download_from_reader, +}; +use nativelink_util::{background_spawn, default_health_status_indicator, tls_utils}; use opentelemetry::context::Context; use opentelemetry::global; use opentelemetry::propagation::Injector; use parking_lot::Mutex; use prost::Message; +use tokio::sync::{Semaphore, oneshot}; use tokio::time::sleep; use tonic::metadata::{Ascii, MetadataKey, MetadataValue}; use tonic::{Code, IntoRequest, Request, Response, Status, Streaming}; -use tracing::{error, trace, warn}; +use tracing::{debug, error, trace, warn}; use uuid::Uuid; struct TonicMetadataInjector<'a>(&'a mut tonic::metadata::MetadataMap); @@ -105,6 +113,101 @@ fn enrich_request( request } +/// Minimum blob size for wire compression when +/// `experimental_remote_cache_compression` is enabled. Below this, zstd +/// framing overhead and per-blob CPU outweigh the wire savings (small blobs +/// are the batching paths' domain). +const WIRE_COMPRESSION_MIN_SIZE_BYTES: u64 = 64 * 1024; + +/// Zstd level for `GrpcStore`'s own compressed transfers. Level 1: measured on +/// artifact-shaped corpora, higher levels bought no meaningful wire reduction +/// while encoding slower, and internal hops are throughput-sensitive. +const WIRE_COMPRESSION_ZSTD_LEVEL: i32 = 1; + +/// Estimated per-entry protobuf and framing overhead charged against +/// `max_batch_bytes`, so that batches of many tiny blobs cannot push a +/// `BatchReadBlobs` response over the gRPC message size limit. +const BATCH_READ_PER_ENTRY_OVERHEAD_BYTES: u64 = 256; + +/// A small-blob read waiting to be coalesced into a `BatchReadBlobs` RPC. +#[derive(Debug)] +struct PendingRead { + digest: DigestInfo, + digest_function: i32, + tx: oneshot::Sender>, +} + +/// The pending small-blob reads plus the total payload bytes they declare. +#[derive(Debug, Default)] +struct ReadQueue { + items: VecDeque, + bytes: u64, +} + +/// State for coalescing small-blob reads into `BatchReadBlobs` RPCs. +/// +/// This uses a slot-based group commit scheme: callers enqueue their read and +/// then try to start a detached dispatcher task by acquiring one of +/// `dispatch_slots` semaphore permits. A dispatcher repeatedly drains up to +/// `max_batch_bytes` worth of pending reads into a single `BatchReadBlobs` +/// request until the queue is empty. This is work-conserving (a read never +/// waits while a dispatch slot is free) and uses no timers. +#[derive(Debug, MetricsComponent)] +struct ReadBatcher { + max_blob_size_bytes: u64, + max_batch_bytes: u64, + max_queued_bytes: u64, + queue: Mutex, + dispatch_slots: Arc, + #[metric(help = "Number of BatchReadBlobs RPCs sent by the read coalescer")] + batches_sent: AtomicU64, + #[metric(help = "Number of blob reads coalesced into BatchReadBlobs RPCs")] + blobs_batched: AtomicU64, + #[metric( + help = "Number of reads that bypassed batching because the queue byte budget was full" + )] + queue_bypasses: AtomicU64, + #[metric(help = "Number of batched reads that resolved to a per-entry error")] + batched_read_errors: AtomicU64, + #[metric(help = "Payload bytes currently waiting in the read coalescer queue")] + queued_bytes: AtomicU64, +} + +impl ReadBatcher { + fn new(config: &GrpcReadBatchingConfig) -> Self { + Self { + max_blob_size_bytes: config.max_blob_size_bytes, + max_batch_bytes: config.max_batch_bytes, + max_queued_bytes: config.max_queued_bytes, + queue: Mutex::new(ReadQueue::default()), + dispatch_slots: Arc::new(Semaphore::new(config.dispatch_slots)), + batches_sent: AtomicU64::new(0), + blobs_batched: AtomicU64::new(0), + queue_bypasses: AtomicU64::new(0), + batched_read_errors: AtomicU64::new(0), + queued_bytes: AtomicU64::new(0), + } + } +} + +/// Mirrors the default retryable-code classification used by +/// `nativelink_util::retry::Retrier::should_retry`: only codes that are +/// always terminal are considered non-retryable. +const fn is_retryable_code(code: Code) -> bool { + !matches!( + code, + Code::Ok + | Code::InvalidArgument + | Code::FailedPrecondition + | Code::OutOfRange + | Code::Unimplemented + | Code::NotFound + | Code::AlreadyExists + | Code::PermissionDenied + | Code::Unauthenticated + ) +} + // This store is usually a pass-through store, but can also be used as a CAS store. Using it as an // AC store has one major side-effect... The has() function may not give the proper size of the // underlying data. This might cause issues if embedded in certain stores. @@ -120,6 +223,14 @@ pub struct GrpcStore { use_legacy_resource_names: bool, headers: Vec<(MetadataKey, MetadataValue)>, forward_headers: Vec, + /// When configured, coalesces small-blob reads into `BatchReadBlobs` + /// RPCs. `None` means reads always use the `ByteStream` `Read` path. + #[metric(group = "read_batcher")] + read_batcher: Option, + remote_cache_compression_enabled: bool, + /// Used by the read coalescer to hand a strong reference of this store + /// to detached dispatcher tasks. + weak_self: Weak, } impl GrpcStore { @@ -146,6 +257,24 @@ impl GrpcStore { let rpc_timeout = Duration::from_secs(spec.rpc_timeout_s); + let read_batcher = match &spec.experimental_read_batching { + Some(config) => { + error_if!( + config.dispatch_slots == 0, + "experimental_read_batching.dispatch_slots must be greater than zero" + ); + // Batched reads share one upstream RPC across many client + // requests, so per-client forwarded headers (e.g. credentials) + // cannot be attached correctly. + error_if!( + !spec.forward_headers.is_empty(), + "experimental_read_batching is incompatible with forward_headers" + ); + Some(ReadBatcher::new(config)) + } + None => None, + }; + let mut headers = Vec::with_capacity(spec.headers.len()); for (name, value) in &spec.headers { // We lowercase keys as HTTP headers are case-insensitive so we should match all cases @@ -161,7 +290,8 @@ impl GrpcStore { headers.push((key, val)); } - Ok(Arc::new(Self { + Ok(Arc::new_cyclic(|weak_self| Self { + weak_self: weak_self.clone(), instance_name: spec.instance_name.clone(), store_type: spec.store_type, retrier: Retrier::new( @@ -178,6 +308,8 @@ impl GrpcStore { ), rpc_timeout, use_legacy_resource_names: spec.use_legacy_resource_names, + read_batcher, + remote_cache_compression_enabled: spec.experimental_remote_cache_compression, headers, // We lowercase keys as HTTP headers are case-insensitive so we should match all cases forward_headers: spec @@ -307,6 +439,217 @@ impl GrpcStore { .await } + /// Enqueues a small-blob read for coalescing into a `BatchReadBlobs` RPC + /// and waits for its result. Returns `None` when the queue is over its + /// byte budget, in which case the caller must fall back to the + /// `ByteStream` `Read` path. + async fn batched_read( + &self, + batcher: &ReadBatcher, + digest: DigestInfo, + ) -> Option> { + // Capture the digest function from the caller's ambient context now; + // the dispatcher runs on a detached task with no such context. + let digest_function: i32 = Context::current() + .get::() + .map_or_else(default_digest_hasher_func, |v| *v) + .proto_digest_func() + .into(); + let (tx, rx) = oneshot::channel(); + { + let mut queue = batcher.queue.lock(); + // Admission control. On overflow gracefully degrade to the + // stream path instead of blocking. + let new_bytes = queue.bytes.saturating_add(digest.size_bytes()); + if new_bytes > batcher.max_queued_bytes { + batcher.queue_bypasses.fetch_add(1, Ordering::Relaxed); + return None; + } + queue.bytes = new_bytes; + batcher.queued_bytes.store(new_bytes, Ordering::Relaxed); + queue.items.push_back(PendingRead { + digest, + digest_function, + tx, + }); + } + + self.maybe_dispatch_read_batches(batcher); + + Some(rx.await.unwrap_or_else(|_| { + Err(make_err!( + Code::Internal, + "Read batch dispatcher dropped result in GrpcStore::batched_read" + )) + })) + } + + /// Tries to start a read batch dispatcher. This is work-conserving: if a + /// dispatch slot is free a dispatcher is started immediately, otherwise + /// one of the active dispatchers is responsible for every currently + /// queued item. The dispatcher runs as a detached task so that + /// cancellation of any individual reader can neither abort an in-flight + /// `BatchReadBlobs` RPC nor strand still-queued waiters. + fn maybe_dispatch_read_batches(&self, batcher: &ReadBatcher) { + let Ok(mut permit) = batcher.dispatch_slots.clone().try_acquire_owned() else { + return; + }; + let Some(store) = self.weak_self.upgrade() else { + return; + }; + background_spawn!("grpc_store_read_batch_dispatch", async move { + let Some(batcher) = &store.read_batcher else { + return; + }; + loop { + store.dispatch_read_batches(batcher).await; + drop(permit); + // Items may have been enqueued between the last drain and + // the permit release. Re-check so they are not stranded + // with no active dispatcher. + if batcher.queue.lock().items.is_empty() { + return; + } + match batcher.dispatch_slots.clone().try_acquire_owned() { + Ok(new_permit) => permit = new_permit, + // Another dispatcher is active and will observe these + // items (or re-check after releasing its own permit). + Err(_) => return, + } + } + }); + } + + /// Drains the pending read queue, sending one `BatchReadBlobs` RPC per + /// drained batch, until the queue is empty. + async fn dispatch_read_batches(&self, batcher: &ReadBatcher) { + loop { + let batch = { + let mut queue = batcher.queue.lock(); + let Some(head) = queue.items.front() else { + return; + }; + // All digests in one BatchReadBlobsRequest must use the same + // digest function. Partition-drain: take items matching the + // head's digest function from anywhere in the queue (up to + // the batch budget) and keep the rest in relative order. + let digest_function = head.digest_function; + let mut batch = Vec::new(); + let mut batch_bytes = 0u64; + let mut rest = VecDeque::with_capacity(queue.items.len()); + while let Some(item) = queue.items.pop_front() { + let item_cost = item + .digest + .size_bytes() + .saturating_add(BATCH_READ_PER_ENTRY_OVERHEAD_BYTES); + if item.digest_function == digest_function + && (batch.is_empty() + || batch_bytes.saturating_add(item_cost) <= batcher.max_batch_bytes) + { + batch_bytes = batch_bytes.saturating_add(item_cost); + queue.bytes = queue.bytes.saturating_sub(item.digest.size_bytes()); + batch.push(item); + } else { + rest.push_back(item); + } + } + queue.items = rest; + batcher.queued_bytes.store(queue.bytes, Ordering::Relaxed); + batch + }; + self.send_read_batch(batcher, batch).await; + } + } + + /// Sends one `BatchReadBlobs` RPC for `batch` and demultiplexes the + /// per-blob responses back to the waiting readers. One failed item does + /// not affect its batch-mates; failure of the whole RPC is broadcast to + /// every item in the batch. + async fn send_read_batch(&self, batcher: &ReadBatcher, batch: Vec) { + let Some(digest_function) = batch.first().map(|item| item.digest_function) else { + return; + }; + // Servers may dedupe duplicate digests within one request, so group + // the waiters per digest and request each digest exactly once, + // fanning the (refcounted) data out to every waiter. + let batch_len = u64::try_from(batch.len()).unwrap_or(u64::MAX); + let mut waiters: HashMap> = HashMap::new(); + for item in batch { + waiters.entry(item.digest).or_default().push(item); + } + let request = BatchReadBlobsRequest { + // batch_read_blobs() overwrites the instance name, so there is + // no need to set it here. + instance_name: String::new(), + digests: waiters.keys().map(|digest| (*digest).into()).collect(), + acceptable_compressors: vec![], + digest_function, + }; + batcher.batches_sent.fetch_add(1, Ordering::Relaxed); + batcher + .blobs_batched + .fetch_add(batch_len, Ordering::Relaxed); + let response = match self.batch_read_blobs(Request::new(request)).await { + Ok(response) => response.into_inner(), + Err(err) => { + // The whole RPC failed, so every waiter in this batch gets + // the error. Waiters may have gone away, ignore send errors. + for item in waiters.into_values().flatten() { + drop(item.tx.send(Err(err.clone()))); + } + return; + } + }; + for entry in response.responses { + let Some(Ok(entry_digest)) = entry.digest.map(DigestInfo::try_from) else { + continue; + }; + let Some(items) = waiters.remove(&entry_digest) else { + continue; + }; + let entry_len = u64::try_from(entry.data.len()).unwrap_or(u64::MAX); + let result = if let Some(status) = entry.status.filter(|status| status.code != 0) { + Err(Error::from(status) + .append("Batch read entry failed in GrpcStore::send_read_batch")) + } else if entry.compressor != 0 { + // We requested no acceptable compressors, so data must be + // returned with the identity compressor. + Err(make_err!( + Code::Internal, + "BatchReadBlobs entry for {entry_digest} used unsupported compressor {}", + entry.compressor + )) + } else if entry_len != entry_digest.size_bytes() { + Err(make_err!( + Code::Internal, + "BatchReadBlobs entry for {entry_digest} returned {entry_len} bytes, expected {}", + entry_digest.size_bytes() + )) + } else { + Ok(entry.data) + }; + if result.is_err() { + batcher.batched_read_errors.fetch_add( + u64::try_from(items.len()).unwrap_or(u64::MAX), + Ordering::Relaxed, + ); + } + for item in items { + drop(item.tx.send(result.clone())); + } + } + // Any waiter with no matching response entry is missing upstream. + for item in waiters.into_values().flatten() { + batcher.batched_read_errors.fetch_add(1, Ordering::Relaxed); + let err = make_err!( + Code::NotFound, + "Blob {} not found in BatchReadBlobs response", + item.digest + ); + drop(item.tx.send(Err(err))); + } + } + pub async fn get_tree( &self, grpc_request: Request, @@ -452,6 +795,19 @@ impl GrpcStore { &self, stream: WriteRequestStreamWrapper, ) -> Result, Error> + where + T: Stream> + Unpin + Send + 'static, + E: Into + 'static, + { + const RESUMABLE: bool = true; + self.write_internal(stream, RESUMABLE).await + } + + async fn write_internal( + &self, + stream: WriteRequestStreamWrapper, + resumable: bool, + ) -> Result, Error> where T: Stream> + Unpin + Send + 'static, E: Into + 'static, @@ -461,10 +817,11 @@ impl GrpcStore { "CAS operation on AC store" ); - let local_state = Arc::new(Mutex::new(WriteState::new( - self.instance_name.clone(), - stream, - ))); + let mut write_state = WriteState::new(self.instance_name.clone(), stream); + if !resumable { + write_state.set_non_resumable(); + } + let local_state = Arc::new(Mutex::new(write_state)); let write_start = std::time::Instant::now(); let instance_name = self.instance_name.clone(); @@ -758,6 +1115,339 @@ impl GrpcStore { .await .map(|_| len) } + + /// Uploads `digest` as a REAPI `compressed-blobs/zstd` write: the raw + /// bytes from `reader` are zstd-encoded on the fly and streamed with + /// compressed write offsets. Used when + /// `experimental_remote_cache_compression` is enabled and the blob meets + /// the size threshold. + async fn update_compressed( + self: Pin<&Self>, + digest: DigestInfo, + reader: DropCloserReadHalf, + ) -> Result { + enum UploadCompletion { + Write(Result<(), Error>), + Encode(Result<(), Error>, Result<(), Error>), + } + + // Compressed writes are NON-resumable: the server-side protocol + // rejects replays from a nonzero compressed offset, so a mid-stream + // failure must surface immediately instead of burning the retry + // budget on guaranteed-rejected resumes. + const NON_RESUMABLE: bool = false; + + struct LocalState { + resource_name: String, + compressed_rx: DropCloserReadHalf, + did_error: bool, + bytes_received: i64, + } + + let mut buf = Uuid::encode_buffer(); + let uuid = Uuid::new_v4().hyphenated().encode_lower(&mut buf); + let resource_name = if self.use_legacy_resource_names { + format!( + "{}/uploads/{}/compressed-blobs/zstd/{}/{}", + &self.instance_name, + uuid, + digest.packed_hash(), + digest.size_bytes(), + ) + } else { + let digest_function = Context::current() + .get::() + .map_or_else(default_digest_hasher_func, |v| *v) + .proto_digest_func() + .as_str_name() + .to_ascii_lowercase(); + format!( + "{}/uploads/{}/compressed-blobs/zstd/{}/{}/{}", + &self.instance_name, + uuid, + digest_function, + digest.packed_hash(), + digest.size_bytes(), + ) + }; + + let (compressed_tx, compressed_rx) = make_buf_channel_pair(); + let mut reader = reader; + let encode_fut = stream_encode_compressed_download_from_reader( + &mut reader, + compressor::Value::Zstd, + WIRE_COMPRESSION_ZSTD_LEVEL, + compressed_tx, + ); + + let local_state = LocalState { + resource_name, + compressed_rx, + did_error: false, + bytes_received: 0, + }; + let stream = Box::pin(unfold(local_state, |mut local_state| async move { + if local_state.did_error { + error!("GrpcStore::update_compressed() polled stream after error was returned"); + return None; + } + let data = match local_state + .compressed_rx + .recv() + .await + .err_tip(|| "In GrpcStore::update_compressed()") + { + Ok(data) => data, + Err(err) => { + local_state.did_error = true; + return Some((Err(err), local_state)); + } + }; + let write_offset = local_state.bytes_received; + local_state.bytes_received += data.len().try_into().unwrap_or(i64::MAX); + Some(( + Ok(WriteRequest { + resource_name: local_state.resource_name.clone(), + write_offset, + finish_write: data.is_empty(), // EOF is when no data was polled. + data, + }), + local_state, + )) + })); + + // The encoder must be driven concurrently with the RPC: the request + // stream's first message is the encoder's first output chunk. + let write_fut = async { + self.write_internal( + WriteRequestStreamWrapper::from(stream) + .await + .err_tip(|| "in GrpcStore::update_compressed()")?, + NON_RESUMABLE, + ) + .await + .map(|_| ()) + .err_tip(|| "in GrpcStore::update_compressed()") + }; + let completion = async { + let write_fut = Box::pin(write_fut); + let encode_fut = Box::pin(encode_fut); + match future::select(write_fut, encode_fut).await { + future::Either::Left((write_result, encode_fut)) => { + drop(encode_fut); + UploadCompletion::Write(write_result) + } + future::Either::Right((encode_result, write_fut)) => { + UploadCompletion::Encode(encode_result, write_fut.await) + } + } + } + .await; + match completion { + UploadCompletion::Write(write_result) => { + write_result?; + // The server settled the write before consuming the whole + // stream (REAPI early completion of a duplicate upload). Do + // not await the encoder: it may be blocked on a stalled + // producer. Cancel it, then drain the raw reader in the + // background so the producer can finish without observing a + // broken pipe from a successful upload. + background_spawn!("grpc_store_compressed_upload_drain", async move { + if let Err(err) = reader.drain().await { + debug!( + ?err, + "Compressed upload reader drain failed after early completion" + ); + } + }); + } + UploadCompletion::Encode(encode_result, write_result) => { + write_result?; + // An encode error with a successful write means the server + // finished without consuming the whole stream; the upload + // itself succeeded. + if let Err(err) = encode_result { + debug!( + ?err, + "Compressed upload encoder ended early after successful write" + ); + } + // The encoder may have stopped because the server completed + // the write while its response stream was still being + // finalized. It has already released its borrow of `reader`, + // so drain any raw input the producer still has in flight. + background_spawn!("grpc_store_compressed_upload_drain", async move { + if let Err(err) = reader.drain().await { + debug!( + ?err, + "Compressed upload reader drain failed after early completion" + ); + } + }); + } + } + Ok(digest.size_bytes()) + } + + /// Reads all of `digest` as a REAPI `compressed-blobs/zstd` read, + /// streaming decode into `writer` with size and digest verification at + /// EOF. Returns `Ok(None)` on success. On a retryable transport failure + /// it returns `Ok(Some(n))` where `n` is the count of decoded bytes + /// already forwarded, so the caller can resume via the identity path at + /// uncompressed offset `n`. Terminal errors (including decode/digest + /// mismatches) propagate as `Err`. + async fn get_part_compressed( + self: Pin<&Self>, + digest: DigestInfo, + writer: &mut DropCloserWriteHalf, + ) -> Result, Error> { + #[derive(Debug)] + enum CompressedReadStage { + Feed, + Decode, + Pump, + } + + let resource_name = if self.use_legacy_resource_names { + format!( + "{}/compressed-blobs/zstd/{}/{}", + &self.instance_name, + digest.packed_hash(), + digest.size_bytes(), + ) + } else { + let digest_function = Context::current() + .get::() + .map_or_else(default_digest_hasher_func, |v| *v) + .proto_digest_func() + .as_str_name() + .to_ascii_lowercase(); + format!( + "{}/compressed-blobs/zstd/{}/{}/{}", + &self.instance_name, + digest_function, + digest.packed_hash(), + digest.size_bytes(), + ) + }; + + let mut stream = match self + .read_internal(ReadRequest { + resource_name, + read_offset: 0, + read_limit: 0, + }) + .await + { + Ok(stream) => stream, + Err(err) if is_retryable_code(err.code) => { + warn!( + ?err, + "Compressed read failed to start, falling back to identity read" + ); + return Ok(Some(0)); + } + Err(err) => return Err(err.append("in GrpcStore::get_part_compressed()")), + }; + + let digest_function = Context::current() + .get::() + .map_or_else(default_digest_hasher_func, |v| *v); + let (mut compressed_tx, compressed_rx) = make_buf_channel_pair(); + let (decoded_tx, mut decoded_rx) = make_buf_channel_pair(); + let decode_fut = stream_decode_compressed_upload( + compressed_rx, + compressor::Value::Zstd, + digest, + digest_function, + decoded_tx, + ); + let feed_fut = async { + loop { + match stream.next().await { + None => { + // A send_eof failure means the decoder already + // settled and dropped its receiver; its result is + // authoritative, so this is not a feed error. + drop(compressed_tx.send_eof()); + return Ok(()); + } + Some(Ok(message)) => { + // Empty chunks are legal on the wire but are the EOF + // marker in buf_channel; skip them. + if !message.data.is_empty() + && compressed_tx.send(message.data).await.is_err() + { + // The decoder stopped consuming (it settled or + // aborted on bad data). Its result decides the + // outcome; reporting a feed error here would + // misclassify a decoder-detected data error as + // retryable transport fallout. + return Ok(()); + } + } + Some(Err(status)) => return Err(Into::::into(status)), + } + } + }; + let forwarded = AtomicU64::new(0); + let pump_fut = async { + loop { + let chunk = decoded_rx + .recv() + .await + .err_tip(|| "in GrpcStore::get_part_compressed()")?; + if chunk.is_empty() { + return writer + .send_eof() + .err_tip(|| "in GrpcStore::get_part_compressed()"); + } + forwarded.fetch_add(chunk.len() as u64, Ordering::Relaxed); + writer + .send(chunk) + .await + .err_tip(|| "in GrpcStore::get_part_compressed()")?; + } + }; + + let result = tokio::try_join!( + async { + feed_fut + .await + .map_err(|err| (CompressedReadStage::Feed, err)) + }, + async { + decode_fut + .await + .map_err(|err| (CompressedReadStage::Decode, err)) + }, + async { + pump_fut + .await + .map_err(|err| (CompressedReadStage::Pump, err)) + }, + ); + + match result { + Ok(((), (), ())) => Ok(None), + Err((CompressedReadStage::Decode, err)) if err.code == Code::InvalidArgument => { + Err(err.append("in GrpcStore::get_part_compressed()")) + } + Err((CompressedReadStage::Pump, err)) if !is_retryable_code(err.code) => Err(err), + Err((CompressedReadStage::Feed, err)) if !is_retryable_code(err.code) => { + Err(err.append("in GrpcStore::get_part_compressed()")) + } + Err((stage, err)) => { + debug!(?stage, ?err, "Compressed read interrupted"); + warn!( + ?err, + forwarded = forwarded.load(Ordering::Relaxed), + "Compressed read interrupted, falling back to identity read" + ); + Ok(Some(forwarded.load(Ordering::Relaxed))) + } + } + } } #[async_trait] @@ -851,6 +1541,12 @@ impl StoreDriver for GrpcStore { return self.update_action_result_from_bytes(digest, reader).await; } + if self.remote_cache_compression_enabled + && digest.size_bytes() >= WIRE_COMPRESSION_MIN_SIZE_BYTES + { + return self.update_compressed(digest, reader).await; + } + let mut buf = Uuid::encode_buffer(); let resource_name = if self.use_legacy_resource_names { format!( @@ -908,7 +1604,7 @@ impl StoreDriver for GrpcStore { }; let write_offset = local_state.bytes_received; - local_state.bytes_received += data.len() as i64; + local_state.bytes_received += data.len().try_into().unwrap_or(i64::MAX); Some(( Ok(WriteRequest { @@ -946,6 +1642,7 @@ impl StoreDriver for GrpcStore { read_limit: i64, } + let is_digest_key = matches!(key, StoreKey::Digest(_)); let digest = key.into_digest(); if matches!(self.store_type, nativelink_config::stores::StoreType::Ac) { let offset = usize::try_from(offset).err_tip(|| "Could not convert offset to usize")?; @@ -963,6 +1660,56 @@ impl StoreDriver for GrpcStore { return writer.send_eof(); } + // When configured, coalesce full reads of small blobs into + // BatchReadBlobs RPCs. `batched_read` returns `None` when the queue + // is over budget, in which case we fall through to the stream path. + if let Some(batcher) = &self.read_batcher + && is_digest_key + && offset == 0 + && length.is_none_or(|len| len >= digest.size_bytes()) + && digest.size_bytes() <= batcher.max_blob_size_bytes + && let Some(result) = self.batched_read(batcher, digest).await + { + match result { + Ok(data) => { + if !data.is_empty() { + writer + .send(data) + .await + .err_tip(|| "Failed to write data in GrpcStore::get_part()")?; + } + return writer + .send_eof() + .err_tip(|| "Failed to send EOF in GrpcStore::get_part()"); + } + // A retryable error falls through to the ByteStream path + // below, which re-enters the full retry machinery. This + // matches the retry behavior reads had before batching. + Err(err) if is_retryable_code(err.code) => { + warn!( + ?err, + "Batched read failed with retryable error, falling back to ByteStream read", + ); + } + Err(err) => return Err(err.append("in GrpcStore::get_part()")), + } + } + + let mut offset = offset; + if self.remote_cache_compression_enabled + && is_digest_key + && offset == 0 + && length.is_none_or(|len| len >= digest.size_bytes()) + && digest.size_bytes() >= WIRE_COMPRESSION_MIN_SIZE_BYTES + { + match self.get_part_compressed(digest, writer).await? { + None => return Ok(()), + // Resume via the identity path from where the compressed + // read left off. + Some(forwarded) => offset = forwarded, + } + } + let resource_name = if self.use_legacy_resource_names { format!( "{}/blobs/{}/{}", @@ -1013,7 +1760,7 @@ impl StoreDriver for GrpcStore { loop { let data = match stream.next().await { // Create an empty response to represent EOF. - None => bytes::Bytes::new(), + None => Bytes::new(), Some(Ok(message)) => message.data, Some(Err(status)) => { return Some(( @@ -1025,7 +1772,8 @@ impl StoreDriver for GrpcStore { )); } }; - let length = data.len() as i64; + let length = data.len().try_into().unwrap_or(i64::MAX); + // This is the usual exit from the loop at EOF. if length == 0 { let eof_result = local_state @@ -1062,10 +1810,7 @@ impl StoreDriver for GrpcStore { self } - fn register_remove_callback( - self: Arc, - _callback: Arc, - ) -> Result<(), Error> { + fn register_remove_callback(self: Arc, _callback: RemoveCallback) -> Result<(), Error> { Err(Error::new( Code::Internal, "gRPC stores are incompatible with removal callbacks".to_string(), diff --git a/nativelink-store/src/lib.rs b/nativelink-store/src/lib.rs index 1ceea189a..b75dd3ca8 100644 --- a/nativelink-store/src/lib.rs +++ b/nativelink-store/src/lib.rs @@ -43,3 +43,4 @@ pub mod shard_store; pub mod size_partitioning_store; pub mod store_manager; pub mod verify_store; +pub mod zstd_store; diff --git a/nativelink-store/src/memory_store.rs b/nativelink-store/src/memory_store.rs index ef7fa6b57..2a71227bc 100644 --- a/nativelink-store/src/memory_store.rs +++ b/nativelink-store/src/memory_store.rs @@ -31,11 +31,11 @@ use nativelink_util::health_utils::{ HealthRegistryBuilder, HealthStatusIndicator, default_health_status_indicator, }; use nativelink_util::store_trait::{ - RemoveItemCallback, StoreDriver, StoreKey, StoreKeyBorrow, StoreOptimizations, UploadSizeInfo, + RemoveCallback, StoreDriver, StoreKey, StoreKeyBorrow, StoreOptimizations, UploadSizeInfo, }; use tracing::warn; -use crate::callback_utils::RemoveItemCallbackHolder; +use crate::callback_utils::RemoveCallbackHolder; use crate::cas_utils::is_zero_digest; #[derive(Clone)] @@ -67,7 +67,7 @@ pub struct MemoryStore { StoreKey<'static>, BytesWrapper, SystemTime, - RemoveItemCallbackHolder, + RemoveCallbackHolder, >, /// The eviction policy's `max_bytes` (0 = unbounded). Cached here so `update` /// can skip writes larger than the entire store budget without buffering @@ -287,12 +287,9 @@ impl StoreDriver for MemoryStore { registry.register_indicator(self); } - fn register_remove_callback( - self: Arc, - callback: Arc, - ) -> Result<(), Error> { + fn register_remove_callback(self: Arc, callback: RemoveCallback) -> Result<(), Error> { self.evicting_map - .add_remove_callback(RemoveItemCallbackHolder::new(callback)); + .add_remove_callback(RemoveCallbackHolder::new(callback)); Ok(()) } } diff --git a/nativelink-store/src/mongo_store.rs b/nativelink-store/src/mongo_store.rs index 3f784c7fd..5492055d3 100644 --- a/nativelink-store/src/mongo_store.rs +++ b/nativelink-store/src/mongo_store.rs @@ -33,7 +33,7 @@ use nativelink_util::buf_channel::{DropCloserReadHalf, DropCloserWriteHalf}; use nativelink_util::health_utils::{HealthRegistryBuilder, HealthStatus, HealthStatusIndicator}; use nativelink_util::spawn; use nativelink_util::store_trait::{ - BoolValue, RemoveItemCallback, SchedulerCurrentVersionProvider, SchedulerIndexProvider, + BoolValue, RemoveCallback, SchedulerCurrentVersionProvider, SchedulerIndexProvider, SchedulerStore, SchedulerStoreDataProvider, SchedulerStoreDecodeTo, SchedulerStoreKeyProvider, SchedulerSubscription, SchedulerSubscriptionManager, StoreDriver, StoreKey, UploadSizeInfo, }; @@ -493,7 +493,7 @@ impl StoreDriver for ExperimentalMongoStore { data.extend_from_slice(&chunk); } - let size = data.len() as i64; + let size = data.len().try_into().unwrap_or(i64::MAX); // Create document let doc = doc! { @@ -627,10 +627,7 @@ impl StoreDriver for ExperimentalMongoStore { registry.register_indicator(self); } - fn register_remove_callback( - self: Arc, - _callback: Arc, - ) -> Result<(), Error> { + fn register_remove_callback(self: Arc, _callback: RemoveCallback) -> Result<(), Error> { // drop because we don't remove anything from Mongo Ok(()) } diff --git a/nativelink-store/src/noop_store.rs b/nativelink-store/src/noop_store.rs index b7aaa7a8d..98b96a505 100644 --- a/nativelink-store/src/noop_store.rs +++ b/nativelink-store/src/noop_store.rs @@ -23,7 +23,7 @@ use nativelink_metric::{ use nativelink_util::buf_channel::{DropCloserReadHalf, DropCloserWriteHalf}; use nativelink_util::health_utils::{HealthStatusIndicator, default_health_status_indicator}; use nativelink_util::store_trait::{ - RemoveItemCallback, StoreDriver, StoreKey, StoreOptimizations, UploadSizeInfo, + RemoveCallback, StoreDriver, StoreKey, StoreOptimizations, UploadSizeInfo, }; #[derive(Debug, Default, Clone, Copy)] @@ -101,10 +101,7 @@ impl StoreDriver for NoopStore { self } - fn register_remove_callback( - self: Arc, - _callback: Arc, - ) -> Result<(), Error> { + fn register_remove_callback(self: Arc, _callback: RemoveCallback) -> Result<(), Error> { // does nothing, so drop Ok(()) } diff --git a/nativelink-store/src/ontap_s3_existence_cache_store.rs b/nativelink-store/src/ontap_s3_existence_cache_store.rs index 264e18660..e238fdb9b 100644 --- a/nativelink-store/src/ontap_s3_existence_cache_store.rs +++ b/nativelink-store/src/ontap_s3_existence_cache_store.rs @@ -36,7 +36,7 @@ use nativelink_util::instant_wrapper::InstantWrapper; use nativelink_util::metrics_utils::CounterWithTime; use nativelink_util::spawn; use nativelink_util::store_trait::{ - RemoveItemCallback, Store, StoreDriver, StoreKey, StoreLike, UploadSizeInfo, + RemoveCallback, RemoveItemCallback, Store, StoreDriver, StoreKey, StoreLike, UploadSizeInfo, }; use serde::{Deserialize, Serialize}; use tokio::fs; @@ -539,10 +539,7 @@ where self } - fn register_remove_callback( - self: Arc, - callback: Arc, - ) -> Result<(), Error> { + fn register_remove_callback(self: Arc, callback: RemoveCallback) -> Result<(), Error> { self.inner_store.register_remove_callback(callback) } } diff --git a/nativelink-store/src/ontap_s3_store.rs b/nativelink-store/src/ontap_s3_store.rs index 42335d0f3..be7868de6 100644 --- a/nativelink-store/src/ontap_s3_store.rs +++ b/nativelink-store/src/ontap_s3_store.rs @@ -47,7 +47,7 @@ use nativelink_util::buf_channel::{ use nativelink_util::health_utils::{HealthStatus, HealthStatusIndicator}; use nativelink_util::instant_wrapper::InstantWrapper; use nativelink_util::retry::{Retrier, RetryResult}; -use nativelink_util::store_trait::{RemoveItemCallback, StoreDriver, StoreKey, UploadSizeInfo}; +use nativelink_util::store_trait::{RemoveCallback, StoreDriver, StoreKey, UploadSizeInfo}; use parking_lot::Mutex; use rustls::{ClientConfig, RootCertStore}; use rustls_pki_types::CertificateDer; @@ -74,8 +74,6 @@ const DEFAULT_MAX_RETRY_BUFFER_PER_REQUEST: usize = 20 * 1024 * 1024; // 20MB // Default limit for concurrent part uploads per multipart upload const DEFAULT_MULTIPART_MAX_CONCURRENT_UPLOADS: usize = 10; -type RemoveCallback = Arc; - #[derive(Debug, MetricsComponent)] pub struct OntapS3Store { s3_client: Arc, @@ -86,7 +84,7 @@ pub struct OntapS3Store { key_prefix: String, retrier: Retrier, #[metric(help = "The number of seconds to consider an object expired")] - consider_expired_after_s: i64, + consider_expired_after_s: u64, #[metric(help = "The number of bytes to buffer for retrying requests")] max_retry_buffer_per_request: usize, #[metric(help = "The number of concurrent uploads allowed for multipart uploads")] @@ -207,7 +205,7 @@ where jitter_fn, spec.common.retry.clone(), ), - consider_expired_after_s: i64::from(spec.common.consider_expired_after_s), + consider_expired_after_s: u64::from(spec.common.consider_expired_after_s), max_retry_buffer_per_request: spec .common .max_retry_buffer_per_request @@ -243,8 +241,12 @@ where if self.consider_expired_after_s != 0 && let Some(last_modified) = head_object_output.last_modified { - let now_s = (self.now_fn)().unix_timestamp() as i64; - if last_modified.secs() + self.consider_expired_after_s <= now_s { + let now_s = (self.now_fn)().unix_timestamp(); + if TryInto::::try_into(last_modified.secs()) + .unwrap_or(u64::MAX) + + self.consider_expired_after_s + <= now_s + { let remove_callbacks = self.remove_callbacks.lock().clone(); let mut callbacks: FuturesUnordered<_> = remove_callbacks .into_iter() @@ -391,7 +393,7 @@ where .put_object() .bucket(&self.bucket) .key(s3_path.clone()) - .content_length(sz as i64) + .content_length(sz.try_into().unwrap_or(i64::MAX)) .body( ByteStream::from(body_bytes) ) @@ -767,10 +769,7 @@ where self } - fn register_remove_callback( - self: Arc, - callback: Arc, - ) -> Result<(), Error> { + fn register_remove_callback(self: Arc, callback: RemoveCallback) -> Result<(), Error> { self.remove_callbacks.lock().push(callback); Ok(()) } diff --git a/nativelink-store/src/redis_store.rs b/nativelink-store/src/redis_store.rs index 78e29075c..f9541984e 100644 --- a/nativelink-store/src/redis_store.rs +++ b/nativelink-store/src/redis_store.rs @@ -35,14 +35,15 @@ use nativelink_error::{Code, Error, ResultExt, make_err, make_input_err}; use nativelink_metric::MetricsComponent; use nativelink_redis_tester::SubscriptionManagerNotify; use nativelink_util::buf_channel::{DropCloserReadHalf, DropCloserWriteHalf}; +use nativelink_util::common::DigestInfo; use nativelink_util::health_utils::{HealthRegistryBuilder, HealthStatus, HealthStatusIndicator}; -use nativelink_util::spawn; use nativelink_util::store_trait::{ - BoolValue, RemoveItemCallback, SchedulerCurrentVersionProvider, SchedulerIndexProvider, + BoolValue, RemoveCallback, SchedulerCurrentVersionProvider, SchedulerIndexProvider, SchedulerStore, SchedulerStoreDataProvider, SchedulerStoreDecodeTo, SchedulerStoreKeyProvider, SchedulerSubscription, SchedulerSubscriptionManager, StoreDriver, StoreKey, UploadSizeInfo, }; use nativelink_util::task::JoinHandleDropGuard; +use nativelink_util::{background_spawn, spawn}; use parking_lot::{Mutex, RwLock}; use patricia_tree::StringPatriciaMap; use redis::aio::{ConnectionLike, ConnectionManager, ConnectionManagerConfig}; @@ -53,9 +54,11 @@ use redis::{ AsyncCommands, AsyncIter, Client, IntoConnectionInfo, PushInfo, ScanOptions, Script, Value, pipe, }; +use serde::Deserialize; +use serde::de::IntoDeserializer; use tokio::select; use tokio::sync::mpsc::{UnboundedReceiver, UnboundedSender, unbounded_channel}; -use tokio::sync::{OwnedSemaphorePermit, Semaphore}; +use tokio::sync::{OnceCell, OwnedSemaphorePermit, Semaphore}; use tokio::time::{sleep, timeout}; use tokio_stream::wrappers::UnboundedReceiverStream; use tracing::{debug, error, info, trace, warn}; @@ -325,6 +328,7 @@ impl RedisManager for StandardRedisManager } async fn psubscribe(&self, pattern: &str) -> Result<(), Error> { + debug!(pattern, "new psubscribe"); let mut connection = self.get_connection().await?.0; let new_subscription = self.subscriptions.lock().insert(String::from(pattern)); if new_subscription { @@ -334,6 +338,7 @@ impl RedisManager for StandardRedisManager } result?; } + debug!(pattern, new_subscription, "new psubscribe complete"); Ok(()) } } @@ -386,10 +391,7 @@ where max_count_per_cursor: u64, /// A manager for subscriptions to keys in Redis. - subscription_manager: tokio::sync::OnceCell>, - - /// Channel for getting subscription messages - subscriber_channel: Mutex>>, + subscription_manager: Arc, /// Permits to limit inflight Redis requests. Technically only /// limits the calls to `get_client()`, but the requests per client @@ -398,6 +400,11 @@ where /// Per-call ceiling for `check_health` PING. health_check_timeout: Duration, + + /// Have we done a subscribe for messages for `remove_callback` subscribes? + has_remove_callback_subscribe: OnceCell<()>, + + remove_callbacks: Arc>>, } impl Debug for RedisStore @@ -416,7 +423,6 @@ where ) .field("scan_count", &self.scan_count) .field("subscription_manager", &self.subscription_manager) - .field("subscriber_channel", &self.subscriber_channel) .field("client_permits", &self.client_permits) .finish() } @@ -447,6 +453,35 @@ impl Drop for ClientWithPermit { } } +/// Decode a `StoreKey` coming from Redis +pub fn decode_key<'a>( + key_prefix: &String, + encoded_key: Cow<'a, str>, +) -> Result, Error> { + let key_no_prefix = if key_prefix.is_empty() { + encoded_key + } else { + match encoded_key.strip_prefix(key_prefix) { + Some(r) => Cow::from(r.to_string()), + None => { + return Err(make_err!( + Code::InvalidArgument, + "Redis key ({}) is missing prefix ({})", + encoded_key, + key_prefix + )); + } + } + }; + let maybe_digest_info: Result<_, serde::de::value::Error> = + DigestInfo::deserialize(key_no_prefix.clone().into_deserializer()); + if let Ok(digest_info) = maybe_digest_info { + Ok(StoreKey::Digest(digest_info)) + } else { + Ok(StoreKey::Str(key_no_prefix)) + } +} + impl RedisStore where C: ConnectionLike + Clone + Sync, @@ -468,6 +503,15 @@ where connection_manager: M, ) -> Result { info!("Redis index fingerprint: {FINGERPRINT_CREATE_INDEX_HEX}"); + let remove_callbacks = Arc::new(async_lock::Mutex::new(Vec::new())); + let subscription_manager = Arc::new(RedisSubscriptionManager::new( + subscriber_channel, + remove_callbacks.clone(), + key_prefix.clone(), + )); + if let Some(channel) = &pub_sub_channel { + connection_manager.psubscribe(channel).await?; + } Ok(Self { connection_manager, @@ -478,11 +522,12 @@ where read_chunk_size, max_chunk_uploads_per_update, scan_count, - subscription_manager: tokio::sync::OnceCell::new(), - subscriber_channel: Mutex::new(Some(subscriber_channel)), + subscription_manager, client_permits: Arc::new(Semaphore::new(max_client_permits)), max_count_per_cursor, health_check_timeout, + has_remove_callback_subscribe: OnceCell::const_new(), + remove_callbacks, }) } @@ -499,8 +544,8 @@ where }) } - /// Encode a [`StoreKey`] so it can be sent to Redis. - fn encode_key<'a>(&self, key: &'a StoreKey<'a>) -> Cow<'a, str> { + /// Encode a `StoreKey` so it can be sent to Redis. + pub fn encode_key<'a>(&self, key: &'a StoreKey<'a>) -> Cow<'a, str> { let key_body = key.as_str(); if self.key_prefix.is_empty() { key_body @@ -913,19 +958,21 @@ where errors.push(format!("Non-utf8 key {raw_key:?}")); continue; }; - if let Some(key) = str_key.strip_prefix(&self.key_prefix) { - let key = StoreKey::new_str(key); - if range.contains(&key) { - iterations += 1; - if !handler(&key) { - error!("Issue in handler"); - errors.push("Issue in handler".to_string()); + match decode_key(&self.key_prefix, Cow::from(str_key)) { + Ok(key) => { + if range.contains(&key) { + iterations += 1; + if !handler(&key) { + error!("Issue in handler"); + errors.push("Issue in handler".to_string()); + } + } else { + trace!(%key, ?range, "Key not in range"); } - } else { - trace!(%key, ?range, "Key not in range"); } - } else { - errors.push("Key doesn't match prefix".to_string()); + Err(e) => { + errors.push(e.to_string()); + } } } Err(err) @@ -1169,13 +1216,13 @@ where // We want to read the data at the key from `offset` to `offset + length`. let data_start = offset; let data_end = data_start - .saturating_add(length.unwrap_or(isize::MAX as usize) as isize) + .saturating_add(length.map_or(isize::MAX, |l| isize::try_from(l).unwrap_or(isize::MAX))) .saturating_sub(1); // And we don't ever want to read more than `read_chunk_size` bytes at a time, so we'll need to iterate. let mut chunk_start = data_start; let mut chunk_end = cmp::min( - data_start.saturating_add(self.read_chunk_size as isize) - 1, + data_start.saturating_add(self.read_chunk_size.try_into().unwrap_or(isize::MAX)) - 1, data_end, ); @@ -1233,7 +1280,8 @@ where // ...and go grab the next chunk. chunk_start = chunk_end + 1; chunk_end = cmp::min( - chunk_start.saturating_add(self.read_chunk_size as isize) - 1, + chunk_start.saturating_add(self.read_chunk_size.try_into().unwrap_or(isize::MAX)) + - 1, data_end, ); } @@ -1293,11 +1341,38 @@ where registry.register_indicator(self); } - fn register_remove_callback( - self: Arc, - _callback: Arc, - ) -> Result<(), Error> { - // As redis doesn't drop stuff, we can just ignore this + fn register_remove_callback(self: Arc, callback: RemoveCallback) -> Result<(), Error> { + debug!(?callback, "New callback"); + let local_self = self.clone(); + background_spawn!("remove_callback_subscribe", async move { + self.remove_callbacks.lock().await.push(callback); + if let Err(err) = local_self.clone().has_remove_callback_subscribe + .get_or_try_init(|| async move { + let mut client = local_self.get_client().await?; + let cfg = redis::cmd("CONFIG").arg("GET").arg("notify-keyspace-events").to_owned().query_async::>(&mut client.connection_manager).await.map_err(|e| Error::from(e).append("Parsing notify-keyspace-events"))?; + if cfg.len() != 1 { + warn!(?cfg, "Got multiple items for CONFIG GET, expected one"); + return Err(make_input_err!("Got multiple items for CONFIG GET, expected one")); + } + let events_cfg = &cfg.first().ok_or_else(|| make_err!(Code::InvalidArgument, "Only one item"))?.1; + if events_cfg.is_empty() { + error!("notify-keyspace-events not enabled for Redis, will fail to get remove callbacks"); + } else if !events_cfg.contains('K') { + error!(notify_keyspace_events=events_cfg, "notify-keyspace-events does not contain 'K' so won't get keyspace events we need for eviction events"); + } else if !events_cfg.contains('A') { + error!(notify_keyspace_events=events_cfg, "notify-keyspace-events does not contain 'A' so we won't get eviction events"); + } + // FIXME: Redis events spec appears unreliable, so we subscribe anyways + // It should just need Ke as per https://redis.io/docs/latest/develop/pubsub/keyspace-notifications/ + // but I'm yet to get reliable eviction events out of that + info!(notify_keyspace_events=events_cfg, "Attempting to subscribe to eviction events"); + self.connection_manager.psubscribe("__key*__:*").await?; + Ok::<(), Error>(()) + }) + .await { + error!(?err, "Error while trying to initialise remove_callback_subscribe"); + } + }); Ok(()) } } @@ -1645,7 +1720,11 @@ pub struct RedisSubscriptionManager { } impl RedisSubscriptionManager { - pub fn new(subscriber_channel: UnboundedReceiver) -> Self { + pub fn new( + subscriber_channel: UnboundedReceiver, + remove_callbacks: Arc>>, + key_prefix: String, + ) -> Self { let subscribed_keys = Arc::new(RwLock::new(StringPatriciaMap::new())); let subscribed_keys_weak = Arc::downgrade(&subscribed_keys); let (tx_for_test, mut rx_for_test) = unbounded_channel(); @@ -1656,6 +1735,7 @@ impl RedisSubscriptionManager { _subscription_spawn: Arc::new(Mutex::new(spawn!( "redis_subscribe_spawn", async move { + debug!("running subscribe loop"); loop { loop { let key = select! { @@ -1682,7 +1762,7 @@ impl RedisSubscriptionManager { error!(?push_info, "Expected exactly 3 values on subscriber channel (pattern, channel, value)"); continue; } - match push_info.data.last().unwrap() { + let value = match push_info.data.last().unwrap() { Value::SimpleString(s) => { s.clone() } @@ -1693,7 +1773,42 @@ impl RedisSubscriptionManager { error!(?other, "Received non-string message in RedisSubscriptionManager"); continue; } + }; + if value == "evicted" { + trace!(?push_info, "Eviction event"); + let eviction_key = if let Some(key) = push_info.data.get(1) { + if let Value::BulkString(s) = key { + String::from_utf8(s.clone()).expect("String message") + } else { + error!(?push_info, "Eviction key wasn't bulk-string"); + continue; + } + } else { + error!(?push_info, "No key in eviction event"); + continue; + }; + trace!(?eviction_key, "Eviction key"); + let Some((_prefix, internal_key)) = eviction_key.split_once(':') else { + error!(?eviction_key, "Eviction key doesn't contain a colon"); + continue; + }; + + let store_key = match decode_key(&key_prefix, Cow::from(internal_key)) { + Ok(k) => k.into_owned(), + Err(err) => { + error!(%err, internal_key, "Bad redis key"); + continue; + } + }; + let locked_remove_callbacks = remove_callbacks.lock().await; + let mut callbacks: FuturesUnordered<_> = + locked_remove_callbacks.iter() + .map(|callback| callback.callback(store_key.borrow())) + .collect(); + while callbacks.next().await.is_some() {} + continue } + value } else { error!("Error receiving message in RedisSubscriptionManager from subscriber_channel"); break; @@ -1786,23 +1901,12 @@ where type SubscriptionManager = RedisSubscriptionManager; async fn subscription_manager(&self) -> Result, Error> { - self.subscription_manager - .get_or_try_init(|| async move { - let Some(subscriber_channel) = self.subscriber_channel.lock().take() else { - return Err(make_input_err!( - "Multiple attempts to obtain the subscription manager in RedisStore" - )); - }; - let Some(pub_sub_channel) = &self.pub_sub_channel else { - return Err(make_input_err!( - "RedisStore must have a pubsub for Redis Scheduler if using subscriptions" - )); - }; - self.connection_manager.psubscribe(pub_sub_channel).await?; - Ok(Arc::new(RedisSubscriptionManager::new(subscriber_channel))) - }) - .await - .map(Clone::clone) + if self.pub_sub_channel.is_none() { + return Err(make_input_err!( + "RedisStore must have a pubsub for Redis Scheduler if using subscriptions" + )); + } + Ok(self.subscription_manager.clone()) } async fn update_data(&self, data: T, expiry: Option) -> Result, Error> diff --git a/nativelink-store/src/ref_store.rs b/nativelink-store/src/ref_store.rs index c6100db8f..26c15678b 100644 --- a/nativelink-store/src/ref_store.rs +++ b/nativelink-store/src/ref_store.rs @@ -23,7 +23,7 @@ use nativelink_metric::MetricsComponent; use nativelink_util::buf_channel::{DropCloserReadHalf, DropCloserWriteHalf}; use nativelink_util::health_utils::{HealthStatusIndicator, default_health_status_indicator}; use nativelink_util::store_trait::{ - RemoveItemCallback, Store, StoreDriver, StoreKey, StoreLike, UploadSizeInfo, + RemoveCallback, Store, StoreDriver, StoreKey, StoreLike, UploadSizeInfo, }; use parking_lot::Mutex; use tracing::{debug, error}; @@ -48,7 +48,7 @@ pub struct RefStore { name: String, store_manager: Weak, inner: StoreReference, - remove_callbacks: Mutex>>, + remove_callbacks: Mutex>, } impl RefStore { @@ -159,10 +159,7 @@ impl StoreDriver for RefStore { self } - fn register_remove_callback( - self: Arc, - callback: Arc, - ) -> Result<(), Error> { + fn register_remove_callback(self: Arc, callback: RemoveCallback) -> Result<(), Error> { self.remove_callbacks.lock().push(callback.clone()); let ref_store = self.inner.cell.0.get(); unsafe { diff --git a/nativelink-store/src/s3_store.rs b/nativelink-store/src/s3_store.rs index 5c2d3837e..f588fc5b0 100644 --- a/nativelink-store/src/s3_store.rs +++ b/nativelink-store/src/s3_store.rs @@ -47,7 +47,7 @@ use nativelink_util::health_utils::{HealthRegistryBuilder, HealthStatus, HealthS use nativelink_util::instant_wrapper::InstantWrapper; use nativelink_util::retry::{Retrier, RetryResult}; use nativelink_util::store_trait::{ - RemoveItemCallback, StoreDriver, StoreKey, StoreOptimizations, UploadSizeInfo, + RemoveCallback, StoreDriver, StoreKey, StoreOptimizations, UploadSizeInfo, }; use parking_lot::Mutex; use tokio::sync::mpsc; @@ -98,7 +98,7 @@ pub struct S3Store { #[metric(help = "The number of concurrent uploads allowed for multipart uploads")] multipart_max_concurrent_uploads: usize, - remove_callbacks: Mutex>>, + remove_callbacks: Mutex>, } impl S3Store @@ -195,7 +195,10 @@ where if self.consider_expired_after_s != 0 && let Some(last_modified) = head_object_output.last_modified { - let now_s = (self.now_fn)().unix_timestamp() as i64; + let now_s = (self.now_fn)() + .unix_timestamp() + .try_into() + .unwrap_or(i64::MAX); if last_modified.secs() + self.consider_expired_after_s <= now_s { let remove_callbacks = self.remove_callbacks.lock().clone(); let mut callbacks: FuturesUnordered<_> = remove_callbacks @@ -323,7 +326,7 @@ where .put_object() .bucket(&self.bucket) .key(s3_path.clone()) - .content_length(sz as i64) + .content_length(sz.try_into().unwrap_or(i64::MAX)) .body(ByteStream::from_body_1_x(BodyWrapper { reader: rx, size: sz, @@ -684,10 +687,7 @@ where registry.register_indicator(self); } - fn register_remove_callback( - self: Arc, - callback: Arc, - ) -> Result<(), Error> { + fn register_remove_callback(self: Arc, callback: RemoveCallback) -> Result<(), Error> { self.remove_callbacks.lock().push(callback); Ok(()) } diff --git a/nativelink-store/src/shard_store.rs b/nativelink-store/src/shard_store.rs index 6855d3a96..2f2494481 100644 --- a/nativelink-store/src/shard_store.rs +++ b/nativelink-store/src/shard_store.rs @@ -27,7 +27,7 @@ use nativelink_metric::MetricsComponent; use nativelink_util::buf_channel::{DropCloserReadHalf, DropCloserWriteHalf}; use nativelink_util::health_utils::{HealthStatusIndicator, default_health_status_indicator}; use nativelink_util::store_trait::{ - RemoveItemCallback, Store, StoreDriver, StoreKey, StoreLike, UploadSizeInfo, + RemoveCallback, Store, StoreDriver, StoreKey, StoreLike, UploadSizeInfo, }; #[derive(Debug, MetricsComponent)] @@ -40,7 +40,7 @@ struct StoreAndWeight { #[derive(Debug, MetricsComponent)] pub struct ShardStore { - // The weights will always be in ascending order a specific store is chosen based on the + // The weights will always be in ascending order a specific store is chosen based on // the hash of the key hash that is nearest-binary searched using the u32 as the index. #[metric( group = "stores", @@ -254,10 +254,7 @@ impl StoreDriver for ShardStore { self } - fn register_remove_callback( - self: Arc, - callback: Arc, - ) -> Result<(), Error> { + fn register_remove_callback(self: Arc, callback: RemoveCallback) -> Result<(), Error> { for store in &self.weights_and_stores { store.store.register_remove_callback(callback.clone())?; } diff --git a/nativelink-store/src/size_partitioning_store.rs b/nativelink-store/src/size_partitioning_store.rs index 883530078..b9883e423 100644 --- a/nativelink-store/src/size_partitioning_store.rs +++ b/nativelink-store/src/size_partitioning_store.rs @@ -23,7 +23,7 @@ use nativelink_metric::MetricsComponent; use nativelink_util::buf_channel::{DropCloserReadHalf, DropCloserWriteHalf}; use nativelink_util::health_utils::{HealthStatusIndicator, default_health_status_indicator}; use nativelink_util::store_trait::{ - RemoveItemCallback, Store, StoreDriver, StoreKey, StoreLike, UploadSizeInfo, + RemoveCallback, Store, StoreDriver, StoreKey, StoreLike, UploadSizeInfo, }; use tokio::join; @@ -171,10 +171,7 @@ impl StoreDriver for SizePartitioningStore { self } - fn register_remove_callback( - self: Arc, - callback: Arc, - ) -> Result<(), Error> { + fn register_remove_callback(self: Arc, callback: RemoveCallback) -> Result<(), Error> { self.lower_store .register_remove_callback(callback.clone())?; self.upper_store.register_remove_callback(callback)?; diff --git a/nativelink-store/src/verify_store.rs b/nativelink-store/src/verify_store.rs index 1b313e08e..f42587aeb 100644 --- a/nativelink-store/src/verify_store.rs +++ b/nativelink-store/src/verify_store.rs @@ -23,11 +23,13 @@ use nativelink_util::buf_channel::{ DropCloserReadHalf, DropCloserWriteHalf, make_buf_channel_pair, }; use nativelink_util::common::PackedHash; -use nativelink_util::digest_hasher::{DigestHasher, digest_hasher_func_from_context}; +use nativelink_util::digest_hasher::{ + DigestHasher, DigestHasherFunc, digest_hasher_func_from_context, +}; use nativelink_util::health_utils::{HealthStatusIndicator, default_health_status_indicator}; use nativelink_util::metrics_utils::CounterWithTime; use nativelink_util::store_trait::{ - RemoveItemCallback, Store, StoreDriver, StoreKey, StoreLike, UploadSizeInfo, + RemoveCallback, Store, StoreDriver, StoreKey, StoreLike, UploadSizeInfo, }; #[derive(Debug, MetricsComponent)] @@ -63,6 +65,7 @@ impl VerifyStore { mut rx: DropCloserReadHalf, maybe_expected_digest_size: Option, original_hash: &PackedHash, + digest_function: Option, mut maybe_hasher: Option<&mut D>, ) -> Result { let mut sum_size: u64 = 0; @@ -119,8 +122,13 @@ impl VerifyStore { let hash_result = digest.packed_hash(); if original_hash != hash_result { self.hash_verification_failures.inc(); + let Some(digest_function) = digest_function else { + return Err(make_input_err!( + "Hash verification failed without a digest function" + )); + }; return Err(make_input_err!( - "Hashes do not match, got: {original_hash} but digest hash was {hash_result}", + "Hash verification using {digest_function} failed: client declared {digest_function}:{original_hash}, but the server computed {digest_function}:{hash_result}", )); } } @@ -182,11 +190,12 @@ impl StoreDriver for VerifyStore { )); } - let mut hasher = if self.verify_hash { - Some(digest_hasher_func_from_context().hasher()) + let digest_function = if self.verify_hash { + Some(digest_hasher_func_from_context()) } else { None }; + let mut hasher = digest_function.map(|digest_function| digest_function.hasher()); let maybe_digest_size = if self.verify_size { Some(digest_size) @@ -201,6 +210,7 @@ impl StoreDriver for VerifyStore { reader, maybe_digest_size, digest.packed_hash(), + digest_function, hasher.as_mut(), ); @@ -235,10 +245,7 @@ impl StoreDriver for VerifyStore { self } - fn register_remove_callback( - self: Arc, - callback: Arc, - ) -> Result<(), Error> { + fn register_remove_callback(self: Arc, callback: RemoveCallback) -> Result<(), Error> { self.inner_store.register_remove_callback(callback) } } diff --git a/nativelink-store/src/zstd_store.rs b/nativelink-store/src/zstd_store.rs new file mode 100644 index 000000000..46df0b58f --- /dev/null +++ b/nativelink-store/src/zstd_store.rs @@ -0,0 +1,1710 @@ +// Copyright 2026 The NativeLink Authors. All rights reserved. +// +// Licensed under the Functional Source License, Version 1.1, Apache 2.0 Future License (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// See LICENSE file for details +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use core::cmp; +use core::future::Future; +use core::pin::Pin; +use core::sync::atomic::{AtomicU64, Ordering}; +use core::time::Duration; +use std::ffi::OsString; +use std::io::{Read, Seek, SeekFrom, Write}; +#[cfg(unix)] +use std::os::unix::fs::PermissionsExt; +use std::sync::{Arc, OnceLock}; + +use async_trait::async_trait; +use bytes::Bytes; +use futures::future::FutureExt; +use nativelink_config::stores::ZstdConfig; +use nativelink_error::{Code, Error, ResultExt, make_err, make_input_err}; +use nativelink_metric::MetricsComponent; +use nativelink_util::buf_channel::{ + BufChannelReader, BufChannelWriter, DropCloserReadHalf, DropCloserWriteHalf, + make_buf_channel_pair, +}; +use nativelink_util::common::DigestInfo; +use nativelink_util::digest_hasher::{ + DigestHasher, DigestHasherFunc, DigestHasherImpl, digest_hasher_func_from_context, +}; +use nativelink_util::fs::{self, FileSlot}; +use nativelink_util::health_utils::{HealthStatusIndicator, default_health_status_indicator}; +use nativelink_util::store_trait::{ + RemoveCallback, Store, StoreDriver, StoreKey, StoreLike, UploadSizeInfo, WireCompressionStore, + WireCompressor, +}; +use nativelink_util::{spawn, spawn_blocking}; +use tokio::io::{AsyncSeekExt, AsyncWriteExt}; +use tokio::sync::{OwnedSemaphorePermit, Semaphore}; + +use crate::cas_utils::is_zero_digest; + +/// Default number of concurrent staged uploads when the config value is `0`. +const DEFAULT_MAX_CONCURRENT_STAGED_UPLOADS: usize = 4; +/// Default number of concurrent identity (uncompressed) reads/writes when the +/// config value is `0`. +const DEFAULT_MAX_CONCURRENT_IDENTITY_OPS: usize = 256; +/// Default number of concurrent recompressions when the config value is `0`. +const DEFAULT_MAX_CONCURRENT_RECOMPRESSIONS: usize = 1; +/// Default total validate-and-stage deadline (seconds) when the config value is +/// `0`. Bounds a client that trickles bytes to hold a staging slot open. +const DEFAULT_STAGE_TIMEOUT_S: u64 = 600; +/// Default commit timeout (seconds) when the config value is `0`. Bounds how +/// long a stalled inner-store commit (or recompression) may hold a staging +/// permit before the upload fails with `DeadlineExceeded` and cleans up. +const DEFAULT_COMMIT_TIMEOUT_S: u64 = 300; +/// Default ceiling for committing a compressed upload straight from memory +/// (no staging file, no `fsync`) when the config value is `0`. +const DEFAULT_MAX_INLINE_COMMIT_SIZE: u64 = 4 * 1024 * 1024; +/// Level used when no `compression_level` is configured, matching the service +/// wire codec (`nativelink-service` `wire_compression::ZSTD_COMPRESSION_LEVEL`). +const DEFAULT_ENCODE_LEVEL: i32 = 3; + +/// The canonical zstd encoding of empty input, computed once. Emitted verbatim +/// for zero-digest `get_zstd` responses so the client always receives a valid +/// zstd stream that decodes to nothing. +fn empty_zstd_frame() -> Bytes { + static EMPTY: OnceLock = OnceLock::new(); + EMPTY + .get_or_init(|| { + Bytes::from( + zstd::bulk::compress(&[], DEFAULT_ENCODE_LEVEL) + .expect("zstd compression of empty input cannot fail"), + ) + }) + .clone() +} + +/// Flattens a `spawn!`/`spawn_blocking!` join result into the task's own result. +fn flatten_join>( + joined: Result, E>, + tip: &'static str, +) -> Result { + joined.err_tip(|| tip)? +} + +/// Maps an `io::Error` from a staging-file operation onto an internal error. +fn stage_file_err(action: &'static str) -> impl FnOnce(std::io::Error) -> Error { + move |e| make_err!(Code::Internal, "Failed to {action} zstd staging file: {e}") +} + +/// Increments a gauge for as long as it is held. +struct Inflight<'a>(&'a AtomicU64); + +impl<'a> Inflight<'a> { + fn enter(gauge: &'a AtomicU64) -> Self { + gauge.fetch_add(1, Ordering::Relaxed); + Self(gauge) + } +} + +impl Drop for Inflight<'_> { + fn drop(&mut self) { + self.0.fetch_sub(1, Ordering::Relaxed); + } +} + +/// Upper bound on the number of compressed bytes a zstd frame can produce for +/// `src_size` uncompressed bytes. Mirrors zstd's `ZSTD_COMPRESSBOUND` with a +/// little extra slack for the frame header/epilogue. Used to bound the inner +/// store upload since the physical size is not known ahead of time. +const fn zstd_compress_bound(src_size: u64) -> u64 { + let low_bound_slack = if src_size < 128 * 1024 { + (128 * 1024 - src_size) >> 11 + } else { + 0 + }; + src_size + .saturating_add(src_size >> 8) + .saturating_add(low_bound_slack) + .saturating_add(64) +} + +/// A blocking source of compressed chunks. An empty chunk signals EOF. +/// +/// This is what lets the streaming and whole-buffer entry points share one +/// bounded decoder: `DropCloserReadHalf` feeds it from a client stream, while +/// [`OneChunk`] feeds it a payload already in memory. +trait ChunkSource { + fn next_chunk(&mut self) -> Result; +} + +impl ChunkSource for DropCloserReadHalf { + fn next_chunk(&mut self) -> Result { + self.blocking_recv() + } +} + +/// Yields one in-memory buffer, then EOF. +struct OneChunk(Option); + +impl ChunkSource for OneChunk { + fn next_chunk(&mut self) -> Result { + Ok(self.0.take().unwrap_or_default()) + } +} + +/// Streaming zstd encode of an identity (raw) upload. +/// +/// The inner store EOF is withheld until the finalized hash and uncompressed +/// size match `digest`; only then is the encoder finished and EOF sent, so a +/// mismatch can never commit the inner upload. +fn encode_identity( + mut reader: DropCloserReadHalf, + tx: DropCloserWriteHalf, + level: i32, + hasher_func: DigestHasherFunc, + digest: DigestInfo, +) -> Result { + let expected_size = digest.size_bytes(); + let mut hasher = hasher_func.hasher(); + let mut uncompressed_size: u64 = 0; + + let mut encoder = + zstd::stream::write::Encoder::new(BufChannelWriter::new(tx), level).map_err(|e| { + make_err!( + Code::Internal, + "Zstd encoder init failed in zstd store: {e}" + ) + })?; + + loop { + let chunk = reader + .blocking_recv() + .err_tip(|| "Failed to read chunk in zstd store update")?; + if chunk.is_empty() { + break; // EOF. + } + + uncompressed_size = uncompressed_size + .checked_add(chunk.len() as u64) + .ok_or_else(|| make_input_err!("Uncompressed size overflow in zstd store update"))?; + if uncompressed_size > expected_size { + return Err(make_input_err!( + "Received more data than digest size in zstd store update, got at least {} but digest says {}", + uncompressed_size, + expected_size + )); + } + + hasher.update(&chunk); + encoder + .write_all(&chunk) + .map_err(|e| make_err!(Code::Internal, "Zstd encode failed in zstd store: {e}"))?; + } + + if uncompressed_size != expected_size { + return Err(make_input_err!( + "Expected size {} but got size {} in zstd store update", + expected_size, + uncompressed_size + )); + } + let actual_digest = hasher.finalize_digest(); + if actual_digest.packed_hash() != digest.packed_hash() { + return Err(make_input_err!( + "Hashes do not match in zstd store update, expected {} but got {}", + digest.packed_hash(), + actual_digest.packed_hash() + )); + } + + let mut writer = encoder.finish().map_err(|e| { + make_err!( + Code::Internal, + "Zstd encoder finish failed in zstd store: {e}" + ) + })?; + writer + .send_eof() + .err_tip(|| "Failed to send EOF in zstd store update")?; + Ok(expected_size) +} + +/// Streaming zstd decode of stored (physical) data back to raw bytes, +/// forwarding at most `length` bytes starting at `offset`. A decode failure of +/// already-stored data is `DataLoss`: it was validated at upload time. +fn decode_identity( + physical_rx: DropCloserReadHalf, + mut raw_tx: DropCloserWriteHalf, + offset: u64, + length: Option, +) -> Result<(), Error> { + let reader = BufChannelReader::new(physical_rx); + let mut decoder = zstd::stream::read::Decoder::new(reader).map_err(|e| { + make_err!( + Code::DataLoss, + "Zstd decoder init failed in zstd store: {e}" + ) + })?; + + let mut to_skip = offset; + let mut remaining = length.unwrap_or(u64::MAX); + let mut buffer = vec![0u8; zstd::zstd_safe::DCtx::out_size()]; + + // The decoder must be drained to EOF even once the byte budget is met: + // dropping `physical_rx` early would fail the still-running inner `get` + // with "receiver disconnected". Mirrors `compression_store::get_part`. + loop { + let read = decoder + .read(&mut buffer) + .map_err(|e| make_err!(Code::DataLoss, "Zstd decode failed in zstd store: {e}"))?; + if read == 0 { + break; // EOF: physical reader fully drained. + } + if remaining == 0 { + continue; // Budget met; keep draining but stop forwarding. + } + + let mut slice = &buffer[..read]; + if to_skip > 0 { + // `slice.len()` bounds the min, so the result always fits in usize. + let skip = usize::try_from(cmp::min(to_skip, slice.len() as u64)).unwrap_or(usize::MAX); + slice = &slice[skip..]; + to_skip -= skip as u64; + } + if slice.is_empty() { + continue; + } + + // `slice.len()` bounds the min, so the result always fits in usize. + let take = usize::try_from(cmp::min(remaining, slice.len() as u64)).unwrap_or(usize::MAX); + remaining -= take as u64; + raw_tx + .blocking_send(Bytes::copy_from_slice(&slice[..take])) + .err_tip(|| "Failed to send decoded chunk in zstd store get_part")?; + } + + raw_tx + .send_eof() + .err_tip(|| "Failed to send decoded EOF in zstd store get_part")?; + Ok(()) +} + +/// Best-effort removal of a staged temp file when dropped. Staged files are pure +/// scratch — the inner store takes the bytes at commit — so the path is removed +/// on success, error, and cancellation alike. +#[derive(Debug)] +struct TempFileGuard { + path: Option, +} + +impl TempFileGuard { + const fn arm(path: String) -> Self { + Self { path: Some(path) } + } +} + +impl Drop for TempFileGuard { + fn drop(&mut self) { + if let Some(path) = &self.path { + // Ignore failures: a backend that commits by moving the file has + // already renamed this path away. + drop(std::fs::remove_file(path)); + } + } +} + +/// A `std::io::Write` sink that hashes and counts decoded output instead of +/// storing it, optionally buffering it for recompression up to `collect_limit`. +/// +/// This is the enforcement point for the decoded-output bound: a `write` that +/// would push `decoded_len` past `max_decoded_size` is rejected *before* any of +/// the offending bytes are hashed or collected. Because the zstd streaming +/// decoder emits output in bounded blocks, a small "zstd bomb" is stopped at the +/// first over-limit block rather than being fully materialized. +struct DecodeSink { + hasher: DigestHasherImpl, + decoded_len: u64, + /// Hard ceiling on total decoded bytes; equals the digest's uncompressed + /// size. + max_decoded_size: u64, + collect: Option>, + collect_limit: u64, +} + +impl DecodeSink { + fn new( + hasher_func: DigestHasherFunc, + max_decoded_size: u64, + collect_limit: Option, + ) -> Self { + Self { + hasher: hasher_func.hasher(), + decoded_len: 0, + max_decoded_size, + collect: collect_limit.map(|_| Vec::new()), + collect_limit: collect_limit.unwrap_or(0), + } + } +} + +impl Write for DecodeSink { + fn write(&mut self, buf: &[u8]) -> std::io::Result { + // `checked_add` (never `saturating_add`) so a length overflow is an + // error, not a silently clamped value. + let new_len = self + .decoded_len + .checked_add(buf.len() as u64) + .ok_or_else(|| { + std::io::Error::new( + std::io::ErrorKind::InvalidData, + "decoded length overflow in zstd store", + ) + })?; + if new_len > self.max_decoded_size { + return Err(std::io::Error::new( + std::io::ErrorKind::InvalidData, + format!( + "decoded output exceeded digest size {} in zstd store", + self.max_decoded_size + ), + )); + } + self.hasher.update(buf); + self.decoded_len = new_len; + if let Some(collected) = self.collect.as_mut() { + if new_len <= self.collect_limit { + collected.extend_from_slice(buf); + } else { + // Too large to re-compress in-memory; stop collecting. + self.collect = None; + } + } + Ok(buf.len()) + } + + fn flush(&mut self) -> std::io::Result<()> { + Ok(()) + } +} + +/// Final check of a bounded decode against the requested digest. The sink +/// already guarantees `decoded_len <= digest.size_bytes()`; this catches the +/// short case and the hash. +fn verify_decoded_digest(sink: &mut DecodeSink, digest: DigestInfo) -> Result<(), Error> { + let expected_size = digest.size_bytes(); + if sink.decoded_len != expected_size { + return Err(make_input_err!( + "Decoded size {} does not match digest size {expected_size} in zstd store", + sink.decoded_len + )); + } + let actual_digest = sink.hasher.finalize_digest(); + if actual_digest.packed_hash() != digest.packed_hash() { + return Err(make_input_err!( + "Hashes do not match in zstd store update, expected {} but got {}", + digest.packed_hash(), + actual_digest.packed_hash() + )); + } + Ok(()) +} + +/// Exclusively create a staged temp file at `path`, returning its open +/// descriptor. `create_new(true)` (`O_CREAT | O_EXCL`) fails if the path already +/// exists or is a symlink, defeating an observe-and-replace race; on unix the +/// owner-only `0o600` mode is applied atomically at creation rather than via a +/// later `chmod` by path. Runs inside `spawn_blocking!`. +fn create_temp_exclusive(path: &str) -> Result { + let mut options = std::fs::OpenOptions::new(); + options.read(true).write(true).create_new(true); + #[cfg(unix)] + { + use std::os::unix::fs::OpenOptionsExt; + options.mode(0o600); + } + options.open(path).map_err(|e| { + make_err!( + Code::Internal, + "Failed to exclusively create zstd staging file {path}: {e}" + ) + }) +} + +/// Decode a complete concatenation of zstd frames from `source` with bounded +/// wire and decoded sizes. `write_wire` receives every compressed chunk before +/// it is decoded (the staging path uses it to persist the exact client bytes). +/// +/// [`zio::Writer::finish`](zstd::stream::zio::Writer::finish) is deliberate: +/// unlike `flush`, it drives the decoder with EOF and rejects an incomplete +/// final frame, so a valid frame followed by a truncated second frame is +/// refused rather than committed. +fn decode_bounded_zstd_stream( + mut source: S, + sink: DecodeSink, + max_compressed_upload_size: u64, + read_context: &'static str, + decode_context: &'static str, + mut write_wire: F, +) -> Result<(DecodeSink, u64), Error> +where + S: ChunkSource, + F: FnMut(&[u8]) -> Result<(), Error>, +{ + let raw_decoder = zstd::stream::raw::Decoder::new().map_err(|e| { + make_err!( + Code::Internal, + "Zstd decoder init failed in zstd store: {e}" + ) + })?; + let mut decoder = zstd::stream::zio::Writer::new(sink, raw_decoder); + let mut wire_bytes_consumed: u64 = 0; + loop { + let chunk = source.next_chunk().err_tip(|| read_context)?; + if chunk.is_empty() { + break; + } + wire_bytes_consumed = wire_bytes_consumed + .checked_add(chunk.len() as u64) + .ok_or_else(|| make_err!(Code::Internal, "Wire byte count overflow in zstd store"))?; + if wire_bytes_consumed > max_compressed_upload_size { + return Err(make_err!( + Code::ResourceExhausted, + "Compressed upload exceeded max_compressed_upload_size ({max_compressed_upload_size} bytes) in zstd store" + )); + } + write_wire(&chunk)?; + decoder + .write_all(&chunk) + .map_err(|e| make_input_err!("{decode_context}: {e}"))?; + } + decoder + .finish() + .map_err(|e| make_input_err!("{decode_context}: {e}"))?; + let (sink, _raw_decoder) = decoder.into_inner(); + Ok((sink, wire_bytes_consumed)) +} + +/// What the blocking staging pass hands back on success. +struct StageOutput { + /// Compressed wire bytes consumed from the client, which is also the size of + /// the staged file. + wire_bytes_consumed: u64, + /// Decoded content, present only when recompression is eligible (decoded + /// length stayed within the configured `max_recompression_size`). + collected: Option>, + /// The descriptor that was validated, rewound to offset 0. + file: FileSlot, + /// The cleanup guard, owned by the blocking task until it returns. + guard: TempFileGuard, +} + +/// Resources transferred to the detached staging task. In particular, the +/// armed cleanup guard stays owned by the task until it returns. +struct StageCompressedInput { + reader: DropCloserReadHalf, + path: String, + digest: DigestInfo, + hasher_func: DigestHasherFunc, + max_compressed_upload_size: u64, + collect_limit: Option, + fs_permit: tokio::sync::SemaphorePermit<'static>, + guard: TempFileGuard, +} + +/// Blocking validation pass: exclusively create the temp file at `path`, stream +/// compressed chunks from `reader` into it while decoding them to recompute the +/// uncompressed length and hash, then check both against `digest`. Enforces the +/// compressed-size cap (`ResourceExhausted`) and, via [`DecodeSink`], the +/// decoded-output cap (`InvalidArgument`) inside the decoder write. +/// +/// The cleanup guard is owned by this detached blocking task until it returns, +/// so cancellation of the async caller cannot leak a file created after the +/// caller stopped awaiting. Only runs from within `spawn_blocking!`. +fn stage_compressed_blocking( + StageCompressedInput { + reader, + path, + digest, + hasher_func, + max_compressed_upload_size, + collect_limit, + fs_permit, + guard, + }: StageCompressedInput, +) -> Result { + let mut file = create_temp_exclusive(&path)?; + + let sink = DecodeSink::new(hasher_func, digest.size_bytes(), collect_limit); + let (mut sink, wire_bytes_consumed) = decode_bounded_zstd_stream( + reader, + sink, + max_compressed_upload_size, + "Failed to read compressed chunk in zstd store staging", + "Zstd decode failed in zstd store staging", + |chunk| file.write_all(chunk).map_err(stage_file_err("write")), + )?; + let collected = sink.collect.take(); + verify_decoded_digest(&mut sink, digest)?; + + file.flush().map_err(stage_file_err("flush"))?; + file.sync_all().map_err(stage_file_err("sync"))?; + // Rewind the *same* descriptor so the commit streams it from the start. + file.seek(SeekFrom::Start(0)) + .map_err(stage_file_err("rewind"))?; + + Ok(StageOutput { + wire_bytes_consumed, + collected, + file: FileSlot::from_std(fs_permit, file), + guard, + }) +} + +/// Blocking bounded decode used to validate that a stream decodes to exactly +/// `expected_size` bytes without buffering the decoded output or touching disk. +/// Returns the compressed wire bytes consumed. +fn validate_bounded_zstd_blocking( + source: S, + hasher_func: DigestHasherFunc, + expected_size: u64, + max_compressed_upload_size: u64, +) -> Result { + let sink = DecodeSink::new(hasher_func, expected_size, None); + let (sink, wire_bytes_consumed) = decode_bounded_zstd_stream( + source, + sink, + max_compressed_upload_size, + "Failed to read chunk validating zero-digest zstd", + "Zstd decode failed for zero digest", + |_| Ok(()), + )?; + if sink.decoded_len != expected_size { + return Err(make_input_err!( + "Zero-digest zstd upload did not decode to empty in zstd store" + )); + } + Ok(wire_bytes_consumed) +} + +/// Blocking whole-buffer validation of a compressed upload: bounded decode, no +/// disk involvement. Returns the wire bytes consumed and, when recompression is +/// eligible, the decoded content. +fn validate_zstd_buffer( + data: Bytes, + digest: DigestInfo, + hasher_func: DigestHasherFunc, + max_compressed_upload_size: u64, + collect_limit: Option, +) -> Result<(u64, Option>), Error> { + let sink = DecodeSink::new(hasher_func, digest.size_bytes(), collect_limit); + let (mut sink, wire_bytes_consumed) = decode_bounded_zstd_stream( + OneChunk(Some(data)), + sink, + max_compressed_upload_size, + "Failed to read compressed buffer in zstd store", + "Zstd decode failed in zstd store inline validation", + |_| Ok(()), + )?; + let collected = sink.collect.take(); + verify_decoded_digest(&mut sink, digest)?; + Ok((wire_bytes_consumed, collected)) +} + +/// A validated staged upload, ready to commit. Dropping it removes the temp file +/// and releases the staging permit. +struct StagedStream { + /// The descriptor validated by staging, rewound to offset 0, taken by the + /// commit. Declared before `_guard` deliberately: Rust drops struct fields + /// in declaration order, closing the descriptor before best-effort path + /// removal on every cancellation/error path. + file: Option, + /// Path of the temp file. Backends that commit by moving the file + /// (`filesystem`) act on this path rather than on the descriptor; see + /// [`ZstdStore::commit_staged`]. + path: String, + /// Physical (compressed) size of what will be committed. + compressed_size: u64, + /// Compressed wire bytes actually consumed from the client. + wire_bytes_consumed: u64, + /// Decoded bytes retained only for optional recompression. Staging bounds + /// this by `max_recompression_size`; it is consumed before commit. + collected: Option>, + _permit: OwnedSemaphorePermit, + _guard: TempFileGuard, +} + +/// A pass-through store that stores blobs zstd-compressed in the inner CAS +/// store while presenting the raw (uncompressed) view to clients. +/// +/// This is CAS-only: only digest keys are supported. Zero-byte digests are +/// never forwarded to the inner store. +/// +/// Zstd implementation selected by `CompressionAlgorithm::Zstd`. +#[derive(MetricsComponent)] +pub struct ZstdStore { + #[metric(group = "inner_store")] + inner_store: Store, + /// Operator-controlled staging directory. Created/permission-checked in + /// `post_init`; used by the zstd fast path. + #[metric(help = "Staging directory for validated compressed uploads")] + temp_path: String, + /// Configured encode level, already validated to `1..=19`; `None` uses + /// [`DEFAULT_ENCODE_LEVEL`]. + compression_level: Option, + /// Hard cap on the number of compressed wire bytes accepted for a single + /// upload before it is rejected with `ResourceExhausted`. + #[metric(help = "Max compressed wire bytes accepted for one upload")] + max_compressed_upload_size: u64, + /// Uncompressed-size ceiling below which an upload is eligible for + /// re-compression at `compression_level`. `0` disables re-compression. + #[metric(help = "Max uncompressed size eligible for recompression")] + max_recompression_size: u64, + /// Compressed uploads at or below this size skip the staging file entirely. + #[metric(help = "Max compressed upload size committed inline from memory")] + max_inline_commit_size: u64, + /// Bounds concurrent staged uploads (temp files being validated at once). + staged_upload_semaphore: Arc, + /// Bounds concurrent identity (uncompressed) reads and writes, each of which + /// occupies a blocking thread for the whole transfer. + identity_semaphore: Arc, + /// Bounds concurrent re-compression passes. Acquired with `try_acquire` so a + /// busy pool skips recompression instead of holding a staging slot. + recompression_semaphore: Arc, + /// Total time one upload may spend being validated and staged, from the + /// moment it is admitted. Unlike a per-message idle timeout, slow continuous + /// progress does not reset it, so a trickling client cannot hold a staging + /// slot open indefinitely. + stage_timeout: Duration, + /// Upper bound on how long the inner-store commit (and any recompression) + /// of a staged upload may take before it fails with `DeadlineExceeded`, + /// releasing the staging permit and removing the temp file. + commit_timeout: Duration, + + #[metric(help = "Compressed uploads accepted through the wire fast path")] + wire_uploads: AtomicU64, + #[metric(help = "Compressed wire bytes accepted through the wire fast path")] + wire_upload_bytes: AtomicU64, + #[metric(help = "Stored zstd streams served byte-for-byte to clients")] + wire_downloads: AtomicU64, + #[metric(help = "Batch reads answered with stored zstd bytes")] + batch_zstd_passthroughs: AtomicU64, + #[metric(help = "Batch reads answered with decoded identity bytes")] + batch_identity_decodes: AtomicU64, + #[metric(help = "Compressed uploads committed inline with no staging file")] + inline_commits: AtomicU64, + #[metric(help = "Compressed uploads committed from a staging file")] + staged_commits: AtomicU64, + #[metric(help = "Compressed uploads currently validating or staging")] + staged_uploads_inflight: AtomicU64, + #[metric(help = "Identity reads and writes currently in flight")] + identity_ops_inflight: AtomicU64, + #[metric(help = "Recompressions that produced a smaller stream and were kept")] + recompressions_applied: AtomicU64, + #[metric(help = "Recompressions that were not smaller and were discarded")] + recompressions_rejected: AtomicU64, + #[metric(help = "Recompressions skipped because every slot was busy")] + recompressions_skipped_busy: AtomicU64, + #[metric(help = "Uploads failed by the validate-and-stage deadline")] + stage_timeouts: AtomicU64, + #[metric(help = "Uploads failed by the recompress-and-commit deadline")] + commit_timeouts: AtomicU64, +} + +impl core::fmt::Debug for ZstdStore { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + f.debug_struct("ZstdStore") + .field("inner_store", &self.inner_store) + .field("compression_level", &self.compression_level) + .finish_non_exhaustive() + } +} + +impl ZstdStore { + pub fn new(spec: &ZstdConfig, inner_store: Store) -> Result, Error> { + if let Some(level) = spec.compression_level + && !(1..=19).contains(&level) + { + return Err(make_input_err!( + "ZstdStore compression_level must be in [1, 19], got {level}" + )); + } + if spec.temp_path.is_empty() { + return Err(make_input_err!("ZstdStore requires a non-empty temp_path")); + } + if spec.max_compressed_upload_size == 0 { + return Err(make_input_err!( + "ZstdStore requires a positive max_compressed_upload_size" + )); + } + // Recompression re-encodes at `compression_level`, so a ceiling without + // a level would silently do nothing. Reject it instead. + if spec.max_recompression_size > 0 && spec.compression_level.is_none() { + return Err(make_input_err!( + "ZstdStore max_recompression_size requires compression_level to be set" + )); + } + let nonzero_or = |configured: usize, default: usize| match configured { + 0 => default, + n => n, + }; + Ok(Arc::new(Self { + inner_store, + temp_path: spec.temp_path.clone(), + compression_level: spec.compression_level, + max_compressed_upload_size: spec.max_compressed_upload_size, + max_recompression_size: spec.max_recompression_size, + max_inline_commit_size: match spec.max_inline_commit_size { + 0 => DEFAULT_MAX_INLINE_COMMIT_SIZE, + n => n, + }, + staged_upload_semaphore: Arc::new(Semaphore::new(nonzero_or( + spec.max_concurrent_staged_uploads, + DEFAULT_MAX_CONCURRENT_STAGED_UPLOADS, + ))), + identity_semaphore: Arc::new(Semaphore::new(nonzero_or( + spec.max_concurrent_identity_ops, + DEFAULT_MAX_CONCURRENT_IDENTITY_OPS, + ))), + recompression_semaphore: Arc::new(Semaphore::new(nonzero_or( + spec.max_concurrent_recompressions, + DEFAULT_MAX_CONCURRENT_RECOMPRESSIONS, + ))), + stage_timeout: Duration::from_secs(match spec.stage_timeout_s { + 0 => DEFAULT_STAGE_TIMEOUT_S, + n => n, + }), + commit_timeout: Duration::from_secs(match spec.commit_timeout_s { + 0 => DEFAULT_COMMIT_TIMEOUT_S, + n => n, + }), + wire_uploads: AtomicU64::new(0), + wire_upload_bytes: AtomicU64::new(0), + wire_downloads: AtomicU64::new(0), + batch_zstd_passthroughs: AtomicU64::new(0), + batch_identity_decodes: AtomicU64::new(0), + inline_commits: AtomicU64::new(0), + staged_commits: AtomicU64::new(0), + staged_uploads_inflight: AtomicU64::new(0), + identity_ops_inflight: AtomicU64::new(0), + recompressions_applied: AtomicU64::new(0), + recompressions_rejected: AtomicU64::new(0), + recompressions_skipped_busy: AtomicU64::new(0), + stage_timeouts: AtomicU64::new(0), + commit_timeouts: AtomicU64::new(0), + })) + } + + #[inline] + fn encode_level(&self) -> i32 { + self.compression_level.unwrap_or(DEFAULT_ENCODE_LEVEL) + } + + /// `Some(limit)` when an upload's decoded bytes should be buffered for a + /// recompression attempt. `new` guarantees a level is configured whenever + /// the ceiling is positive. + #[inline] + const fn recompression_limit(&self) -> Option { + if self.max_recompression_size > 0 { + Some(self.max_recompression_size) + } else { + None + } + } + + async fn acquire_staging_permit(&self) -> Result { + self.staged_upload_semaphore + .clone() + .acquire_owned() + .await + .map_err(|e| make_err!(Code::Internal, "Staged upload semaphore closed: {e}")) + } + + async fn acquire_identity_permit(&self) -> Result { + self.identity_semaphore + .clone() + .acquire_owned() + .await + .map_err(|e| make_err!(Code::Internal, "Identity semaphore closed: {e}")) + } + + async fn update_identity( + self: Pin<&Self>, + key: StoreKey<'_>, + mut reader: DropCloserReadHalf, + ) -> Result { + let StoreKey::Digest(digest) = key else { + return Err(make_input_err!( + "ZstdStore only supports digest keys, got {key:?}" + )); + }; + + if is_zero_digest(digest) { + return reader.recv().await.and_then(|chunk| { + if chunk.is_empty() { + Ok(0) + } else { + Err(make_err!(Code::Internal, "Zero byte hash was not empty")) + } + }); + } + + // The encode below parks a blocking thread for the whole upload, so it + // must be admitted rather than allowed to exhaust the shared pool. + let _permit = self.acquire_identity_permit().await?; + let _inflight = Inflight::enter(&self.identity_ops_inflight); + + let (tx, rx) = make_buf_channel_pair(); + let inner_store = self.inner_store.clone(); + let max_output_size = zstd_compress_bound(digest.size_bytes()); + let update_fut = spawn!("zstd_store_update_spawn", async move { + inner_store + .update(digest, rx, UploadSizeInfo::MaxSize(max_output_size)) + .await + .err_tip(|| "Inner store update in zstd store failed") + }) + .map(|joined| flatten_join(joined, "Failed to run zstd store update spawn")); + + let level = self.encode_level(); + let hasher_func = digest_hasher_func_from_context(); + let encode_fut = spawn_blocking!("zstd_store_encode_identity", move || { + encode_identity(reader, tx, level, hasher_func, digest) + }) + .map(|joined| flatten_join(joined, "Failed to run zstd store encode task")); + + let (encode_res, update_res) = tokio::join!(encode_fut, update_fut); + match (encode_res, update_res) { + (Ok(size), Ok(_)) => Ok(size), + (Err(e), _) | (_, Err(e)) => Err(e), + } + } + + async fn get_part_identity( + self: Pin<&Self>, + key: StoreKey<'_>, + writer: &mut DropCloserWriteHalf, + offset: u64, + length: Option, + ) -> Result<(), Error> { + let StoreKey::Digest(digest) = key else { + return Err(make_input_err!( + "ZstdStore only supports digest keys, got {key:?}" + )); + }; + + if is_zero_digest(digest) { + writer + .send_eof() + .err_tip(|| "Failed to send zero-digest EOF in zstd store get_part")?; + return Ok(()); + } + + // The decode below parks a blocking thread until the client has consumed + // the whole blob, so it must be admitted like an upload. + let _permit = self.acquire_identity_permit().await?; + let _inflight = Inflight::enter(&self.identity_ops_inflight); + + let (physical_tx, physical_rx) = make_buf_channel_pair(); + let inner_store = self.inner_store.clone(); + let get_fut = spawn!("zstd_store_get_part_spawn", async move { + inner_store + .get_part(digest, physical_tx, 0, None) + .await + .err_tip(|| "Inner store get in zstd store failed") + }) + .map(|joined| flatten_join(joined, "Failed to run zstd store get spawn")); + + let (raw_tx, mut raw_rx) = make_buf_channel_pair(); + let decode_fut = spawn_blocking!("zstd_store_decode_identity", move || { + decode_identity(physical_rx, raw_tx, offset, length) + }) + .map(|joined| flatten_join(joined, "Failed to run zstd store decode task")); + + let pump_fut = async move { + loop { + let chunk = raw_rx + .recv() + .await + .err_tip(|| "Failed to read decoded chunk in zstd store get_part")?; + if chunk.is_empty() { + break; + } + writer + .send(chunk) + .await + .err_tip(|| "Failed to send chunk in zstd store get_part")?; + } + writer + .send_eof() + .err_tip(|| "Failed to send EOF in zstd store get_part")?; + Result::<(), Error>::Ok(()) + }; + + let (get_res, decode_res, pump_res) = tokio::join!(get_fut, decode_fut, pump_fut); + // Prioritize the inner get error (e.g. NotFound) as the root cause, then + // the decode error (DataLoss on corrupt data), then the pump error. + get_res?; + decode_res?; + pump_res?; + Ok(()) + } + + /// Byte-for-byte passthrough of the stored zstd stream. Always emits zstd: + /// zero digests are answered with the canonical empty encoding. + pub async fn get_zstd( + &self, + digest: DigestInfo, + mut writer: DropCloserWriteHalf, + ) -> Result<(), Error> { + if is_zero_digest(digest) { + writer + .send(empty_zstd_frame()) + .await + .err_tip(|| "Failed to send empty zstd in zstd store get_zstd")?; + writer + .send_eof() + .err_tip(|| "Failed to send EOF in zstd store get_zstd")?; + return Ok(()); + } + // The physical bytes are already zstd; pipe them straight through. + self.inner_store + .get_part(digest, writer, 0, None) + .await + .err_tip(|| "Inner store get_part in zstd store get_zstd failed")?; + self.wire_downloads.fetch_add(1, Ordering::Relaxed); + Ok(()) + } + + /// Accept a client-supplied compressed (zstd) stream: validate, stage, + /// optionally re-compress, then commit. Returns the number of compressed + /// wire bytes consumed. + pub async fn update_zstd( + &self, + digest: DigestInfo, + digest_function: DigestHasherFunc, + reader: DropCloserReadHalf, + ) -> Result { + if is_zero_digest(digest) { + return self.validate_empty_zstd(digest_function, reader).await; + } + let wire_bytes = self + .update_staged_zstd(digest, digest_function, reader) + .await?; + self.record_wire_upload(wire_bytes); + Ok(wire_bytes) + } + + /// Whole-`Bytes` variant of [`ZstdStore::update_zstd`] for `BatchUpdate`. + pub async fn update_zstd_oneshot( + &self, + digest: DigestInfo, + digest_function: DigestHasherFunc, + data: Bytes, + ) -> Result { + // Zero digests must not be whole-buffer decoded by an allocating + // decoder, so they go through the same bounded decoder as everything + // else — just fed from memory instead of a channel. + if is_zero_digest(digest) { + return self + .validate_empty_zstd(digest_function, OneChunk(Some(data))) + .await; + } + + let wire_bytes = if data.len() as u64 <= self.max_inline_commit_size { + self.update_inline_zstd(digest, digest_function, data) + .await? + } else { + self.update_large_oneshot_zstd(digest, digest_function, data) + .await? + }; + self.record_wire_upload(wire_bytes); + Ok(wire_bytes) + } + + fn record_wire_upload(&self, wire_bytes: u64) { + self.wire_uploads.fetch_add(1, Ordering::Relaxed); + self.wire_upload_bytes + .fetch_add(wire_bytes, Ordering::Relaxed); + } + + /// Batch read selection: return `(data, is_zstd)`. Prefers the physical + /// zstd bytes when the client accepts zstd and compression actually helped; + /// otherwise decodes to raw. + pub async fn get_for_batch( + &self, + digest: DigestInfo, + client_accepts_zstd: bool, + ) -> Result<(Bytes, bool), Error> { + if is_zero_digest(digest) { + return Ok((Bytes::new(), false)); + } + let physical = self + .inner_store + .get_part_unchunked(digest, 0, None) + .await + .err_tip(|| "Inner store get in zstd store get_for_batch failed")?; + if client_accepts_zstd && (physical.len() as u64) < digest.size_bytes() { + self.batch_zstd_passthroughs.fetch_add(1, Ordering::Relaxed); + return Ok((physical, true)); + } + let expected_size = usize::try_from(digest.size_bytes()) + .map_err(|_| make_err!(Code::Internal, "Digest size too large for this platform"))?; + let raw = flatten_join( + spawn_blocking!("zstd_store_batch_decode", move || { + // Stored data was validated at upload time, and + // `bulk::decompress` bounds its output by `expected_size`, so a + // failure here is corruption of already-stored bytes. + zstd::bulk::decompress(&physical, expected_size) + .map_err(|e| make_err!(Code::DataLoss, "Zstd decode failed in zstd store: {e}")) + }) + .await, + "Failed to run zstd store batch decode task", + )?; + self.batch_identity_decodes.fetch_add(1, Ordering::Relaxed); + Ok((Bytes::from(raw), false)) + } + + /// Validate that a zero-digest client stream decodes to empty without + /// touching the inner store or allocating the decoded output. Takes a + /// staging permit and the staging deadline, so a flood of zero-digest + /// streams cannot spawn unbounded blocking decode jobs or park them. + /// Returns the compressed wire bytes consumed. + async fn validate_empty_zstd( + &self, + digest_function: DigestHasherFunc, + source: S, + ) -> Result { + let _permit = self.acquire_staging_permit().await?; + let _inflight = Inflight::enter(&self.staged_uploads_inflight); + let max_compressed_upload_size = self.max_compressed_upload_size; + let validate_fut = spawn_blocking!("zstd_store_validate_empty", move || { + validate_bounded_zstd_blocking(source, digest_function, 0, max_compressed_upload_size) + }); + match tokio::time::timeout(self.stage_timeout, validate_fut).await { + Ok(joined) => flatten_join(joined, "Failed to run zstd store empty validation task"), + Err(_elapsed) => Err(self.stage_timeout_err()), + } + } + + fn stage_timeout_err(&self) -> Error { + self.stage_timeouts.fetch_add(1, Ordering::Relaxed); + make_err!( + Code::DeadlineExceeded, + "zstd store validation and staging exceeded stage_timeout_s ({}s); cleaning up", + self.stage_timeout.as_secs() + ) + } + + /// Bound the post-validation recompression and commit of an already-staged + /// or already-validated upload. + async fn with_commit_timeout(&self, commit: F) -> Result + where + F: Future>, + { + match tokio::time::timeout(self.commit_timeout, commit).await { + Ok(result) => result, + Err(_elapsed) => { + self.commit_timeouts.fetch_add(1, Ordering::Relaxed); + Err(make_err!( + Code::DeadlineExceeded, + "zstd store recompression or commit timed out after {}s; cleaning up staged files", + self.commit_timeout.as_secs() + )) + } + } + } + + /// Validate and stage a non-zero-digest compressed stream, then commit it. + /// The blocking staging task owns its cleanup guard, so cancellation while + /// it is detached still removes its temp file once it stops. + async fn update_staged_zstd( + &self, + digest: DigestInfo, + digest_function: DigestHasherFunc, + reader: DropCloserReadHalf, + ) -> Result { + let mut staged = self + .stage_compressed(digest, digest_function, reader) + .await?; + let wire_bytes_consumed = staged.wire_bytes_consumed; + self.with_commit_timeout(async { + self.recompress_staged(&mut staged).await?; + self.commit_staged(digest, staged).await?; + Ok(wire_bytes_consumed) + }) + .await + } + + /// Oneshot payload too large to hold a second copy of in memory: feed it + /// through the same bounded streaming staging path as a client stream. + async fn update_large_oneshot_zstd( + &self, + digest: DigestInfo, + digest_function: DigestHasherFunc, + data: Bytes, + ) -> Result { + let (mut tx, rx) = make_buf_channel_pair(); + let feed_fut = spawn!("zstd_store_oneshot_feed", async move { + if !data.is_empty() { + tx.send(data) + .await + .err_tip(|| "Failed to feed oneshot data in zstd store")?; + } + tx.send_eof() + .err_tip(|| "Failed to send oneshot EOF in zstd store")?; + Result::<(), Error>::Ok(()) + }); + + let stage_res = self.update_staged_zstd(digest, digest_function, rx).await; + let feed_res = flatten_join(feed_fut.await, "Failed to run zstd store oneshot feed task"); + // Staging owns the meaningful error: rejecting an upload drops the + // reader, which makes the feeder fail too. Only surface the feeder's + // error when staging itself succeeded. + let wire_bytes = stage_res?; + feed_res?; + Ok(wire_bytes) + } + + /// Validate and commit a small compressed upload straight from memory: no + /// staging file and no `fsync`. `BatchUpdateBlobs` payloads are small and + /// numerous, so a per-blob disk round trip would dominate their cost. + async fn update_inline_zstd( + &self, + digest: DigestInfo, + digest_function: DigestHasherFunc, + data: Bytes, + ) -> Result { + let _permit = self.acquire_staging_permit().await?; + let _inflight = Inflight::enter(&self.staged_uploads_inflight); + + let max_compressed_upload_size = self.max_compressed_upload_size; + let collect_limit = self.recompression_limit(); + // `Bytes` is refcounted, so the validation copy is free. + let to_validate = data.clone(); + let validate_fut = spawn_blocking!("zstd_store_validate_inline", move || { + validate_zstd_buffer( + to_validate, + digest, + digest_function, + max_compressed_upload_size, + collect_limit, + ) + }); + let (wire_bytes_consumed, collected) = + match tokio::time::timeout(self.stage_timeout, validate_fut).await { + Ok(joined) => flatten_join(joined, "Failed to run zstd store inline validation")?, + Err(_elapsed) => return Err(self.stage_timeout_err()), + }; + + self.with_commit_timeout(async { + let mut payload = data; + if let Some(collected) = collected + && let Some(smaller) = self + .maybe_recompress(collected, payload.len() as u64) + .await? + { + payload = Bytes::from(smaller); + } + self.inner_store + .update_oneshot(digest, payload) + .await + .err_tip(|| "Failed to commit inline zstd upload to inner store")?; + self.inline_commits.fetch_add(1, Ordering::Relaxed); + Ok(wire_bytes_consumed) + }) + .await + } + + /// Stage a validated compressed upload to a temp file. The returned + /// [`StagedStream`] owns the cleanup guard and staging permit. + async fn stage_compressed( + &self, + digest: DigestInfo, + digest_function: DigestHasherFunc, + reader: DropCloserReadHalf, + ) -> Result { + let permit = self.acquire_staging_permit().await?; + let _inflight = Inflight::enter(&self.staged_uploads_inflight); + + let stage_path = format!("{}/zstd-stage-{}", self.temp_path, uuid::Uuid::new_v4()); + // Transfer an already-armed guard to the detached blocking task. If this + // async future is cancelled (including by `stage_timeout`) while that + // task is still creating or validating the file, the task keeps owning + // cleanup until it finishes; the guard comes back only on success. + let guard = TempFileGuard::arm(stage_path.clone()); + + let fs_permit = fs::get_permit().await?; + let max_compressed_upload_size = self.max_compressed_upload_size; + let collect_limit = self.recompression_limit(); + let blocking_path = stage_path.clone(); + let stage_fut = spawn_blocking!("zstd_store_stage", move || { + stage_compressed_blocking(StageCompressedInput { + reader, + path: blocking_path, + digest, + hasher_func: digest_function, + max_compressed_upload_size, + collect_limit, + fs_permit, + guard, + }) + }); + let output = match tokio::time::timeout(self.stage_timeout, stage_fut).await { + Ok(joined) => flatten_join(joined, "Failed to run zstd store staging task")?, + Err(_elapsed) => return Err(self.stage_timeout_err()), + }; + + Ok(StagedStream { + file: Some(output.file), + path: stage_path, + compressed_size: output.wire_bytes_consumed, + wire_bytes_consumed: output.wire_bytes_consumed, + collected: output.collected, + _permit: permit, + _guard: output.guard, + }) + } + + /// Re-encode `collected` at the configured level, returning it only if it is + /// smaller than `current_size`. + /// + /// Best-effort by design: an upload that finds every recompression slot busy + /// returns `None` rather than queueing, because it is holding a staging + /// permit and queueing here would let a small recompression pool throttle + /// the whole upload path. + async fn maybe_recompress( + &self, + collected: Vec, + current_size: u64, + ) -> Result>, Error> { + let Ok(_rec_permit) = self.recompression_semaphore.clone().try_acquire_owned() else { + self.recompressions_skipped_busy + .fetch_add(1, Ordering::Relaxed); + return Ok(None); + }; + let level = self.encode_level(); + let recompressed = flatten_join( + spawn_blocking!("zstd_store_recompress", move || { + zstd::bulk::compress(&collected, level).map_err(|e| { + make_err!( + Code::Internal, + "Zstd re-compression failed in zstd store: {e}" + ) + }) + }) + .await, + "Failed to run zstd store recompress task", + )?; + if (recompressed.len() as u64) < current_size { + self.recompressions_applied.fetch_add(1, Ordering::Relaxed); + Ok(Some(recompressed)) + } else { + self.recompressions_rejected.fetch_add(1, Ordering::Relaxed); + Ok(None) + } + } + + /// Overwrite the staged file with a smaller re-encoding when one is + /// available. The validated descriptor stays open and is reused, so this + /// needs neither a second file permit (which could deadlock under + /// exhaustion) nor a reopen of the staged pathname. + async fn recompress_staged(&self, staged: &mut StagedStream) -> Result<(), Error> { + let Some(collected) = staged.collected.take() else { + return Ok(()); + }; + let Some(recompressed) = self + .maybe_recompress(collected, staged.compressed_size) + .await? + else { + return Ok(()); + }; + + let file = staged + .file + .as_mut() + .ok_or_else(|| make_err!(Code::Internal, "Staged zstd file already consumed"))?; + file.as_ref() + .set_len(0) + .await + .map_err(stage_file_err("truncate"))?; + file.seek(SeekFrom::Start(0)) + .await + .map_err(stage_file_err("rewind"))?; + file.write_all(&recompressed) + .await + .map_err(stage_file_err("write"))?; + file.flush().await.map_err(stage_file_err("flush"))?; + file.as_ref() + .sync_all() + .await + .map_err(stage_file_err("sync"))?; + file.seek(SeekFrom::Start(0)) + .await + .map_err(stage_file_err("rewind"))?; + staged.compressed_size = recompressed.len() as u64; + Ok(()) + } + + /// Commit a validated staged upload to the inner store. Only called after + /// validation succeeded, so the inner store never sees corrupt data. + /// Consumes `staged`, so the staging permit and temp file are released and + /// removed once the commit resolves. + /// + /// Both the validated descriptor and its pathname are handed over, and which + /// one the backend uses is backend-specific: stores that stream the bytes + /// (the default `StoreDriver::update_with_whole_file`, so `memory`, S3, …) + /// read the exact descriptor that was validated, while `filesystem` drops it + /// and commits by `rename(2)` on the path. For the latter, the guarantee + /// against an observe-and-replace race comes from `O_EXCL` creation of an + /// unguessable name inside an operator-private `temp_path` — enforced by + /// `post_init` — not from descriptor pinning. That backend also requires + /// `temp_path` and its `content_path` to share a filesystem; a cross-device + /// `rename` fails with `EXDEV` and the upload is rejected. + async fn commit_staged( + &self, + digest: DigestInfo, + mut staged: StagedStream, + ) -> Result<(), Error> { + let file = staged + .file + .take() + .ok_or_else(|| make_err!(Code::Internal, "Staged zstd file already consumed"))?; + let compressed_size = staged.compressed_size; + let path = OsString::from(staged.path.clone()); + self.inner_store + .update_with_whole_file( + digest, + path, + file, + UploadSizeInfo::ExactSize(compressed_size), + ) + .await + .err_tip(|| "Failed to commit staged zstd upload to inner store")?; + self.staged_commits.fetch_add(1, Ordering::Relaxed); + Ok(()) + // `staged` (permit + cleanup guard) drops here on every path. + } +} + +#[async_trait] +impl WireCompressionStore for ZstdStore { + async fn update_compressed( + self: Arc, + digest: DigestInfo, + digest_function: DigestHasherFunc, + compressor: WireCompressor, + reader: DropCloserReadHalf, + ) -> Result { + match compressor { + WireCompressor::Zstd => self.update_zstd(digest, digest_function, reader).await, + } + } + + async fn get_compressed( + self: Arc, + digest: DigestInfo, + compressor: WireCompressor, + writer: DropCloserWriteHalf, + ) -> Result<(), Error> { + match compressor { + WireCompressor::Zstd => self.get_zstd(digest, writer).await, + } + } + + async fn update_compressed_oneshot( + self: Arc, + digest: DigestInfo, + digest_function: DigestHasherFunc, + compressor: WireCompressor, + data: Bytes, + ) -> Result<(), Error> { + match compressor { + WireCompressor::Zstd => self + .update_zstd_oneshot(digest, digest_function, data) + .await + .map(|_| ()), + } + } + + async fn get_for_batch( + self: Arc, + digest: DigestInfo, + acceptable_compressors: &[WireCompressor], + ) -> Result<(Bytes, Option), Error> { + let accepts_zstd = acceptable_compressors.contains(&WireCompressor::Zstd); + let (data, is_zstd) = Self::get_for_batch(self.as_ref(), digest, accepts_zstd).await?; + Ok((data, is_zstd.then_some(WireCompressor::Zstd))) + } +} + +#[async_trait] +impl StoreDriver for ZstdStore { + async fn post_init(self: Arc) -> Result<(), Error> { + tokio::fs::create_dir_all(&self.temp_path) + .await + .map_err(|e| { + make_err!( + Code::Internal, + "Failed to create ZstdStore temp_path {}: {e}", + self.temp_path + ) + })?; + let meta = tokio::fs::metadata(&self.temp_path).await.map_err(|e| { + make_err!( + Code::Internal, + "Failed to stat ZstdStore temp_path {}: {e}", + self.temp_path + ) + })?; + if !meta.is_dir() { + return Err(make_input_err!( + "ZstdStore temp_path {} is not a directory", + self.temp_path + )); + } + #[cfg(unix)] + { + // Staged files hold validated-but-uncommitted blob contents, and a + // `filesystem` backend commits them by pathname, so the directory + // itself must not be writable by untrusted local users. A sticky + // world-writable dir (like /tmp) restricts deletes/renames to the + // owner, so it is tolerated. + let mode = meta.permissions().mode(); + if mode & 0o002 != 0 && mode & 0o1000 == 0 { + return Err(make_input_err!( + "ZstdStore temp_path {} is world-writable without the sticky bit (mode {:o}); \ + use an operator-private directory not writable by untrusted users", + self.temp_path, + mode & 0o7777 + )); + } + } + // Probe writability at startup rather than discovering it on the first + // upload. + let probe_path = format!( + "{}/.zstd-store-probe-{}", + self.temp_path, + uuid::Uuid::new_v4() + ); + tokio::fs::write(&probe_path, [0u8]).await.map_err(|e| { + make_err!( + Code::Internal, + "ZstdStore temp_path {} is not writable: {e}", + self.temp_path + ) + })?; + tokio::fs::remove_file(&probe_path).await.map_err(|e| { + make_err!( + Code::Internal, + "Failed to remove ZstdStore write-probe file {probe_path}: {e}" + ) + })?; + self.inner_store.clone().into_inner().post_init().await + } + + fn wire_compression_store(self: Arc) -> Option> { + Some(self) + } + + async fn has_with_results( + self: Pin<&Self>, + keys: &[StoreKey<'_>], + results: &mut [Option], + ) -> Result<(), Error> { + // ZstdStore is CAS-only. + for key in keys { + if let StoreKey::Str(_) = key { + return Err(make_input_err!("ZstdStore only supports digest keys")); + } + } + + // Invariant: zero digests never touch the inner store. Satisfy them + // directly and only forward the non-zero keys to the inner store. + for (key, result) in keys.iter().zip(results.iter_mut()) { + if is_zero_digest(key.borrow()) { + *result = Some(0); + } + } + + let nonzero_keys = keys + .iter() + .filter(|key| !is_zero_digest(key.borrow())) + .map(StoreKey::borrow) + .collect::>(); + if nonzero_keys.is_empty() { + return Ok(()); + } + + let mut nonzero_results = vec![None; nonzero_keys.len()]; + self.inner_store + .as_store_driver_pin() + .has_with_results(&nonzero_keys, &mut nonzero_results) + .await?; + + // Presence comes from the inner store, but the reported size is always + // the digest's uncompressed size (the physical zstd size is meaningless + // to clients). + let nonzero_slots = keys + .iter() + .zip(results.iter_mut()) + .filter_map(|(key, result)| (!is_zero_digest(key.borrow())).then_some((key, result))); + for ((key, result), inner_result) in nonzero_slots.zip(nonzero_results) { + *result = match (inner_result, key) { + (Some(_), StoreKey::Digest(digest)) => Some(digest.size_bytes()), + (other, _) => other, + }; + } + Ok(()) + } + + async fn update( + self: Pin<&Self>, + key: StoreKey<'_>, + reader: DropCloserReadHalf, + _upload_size: UploadSizeInfo, + ) -> Result { + self.update_identity(key, reader).await + } + + async fn get_part( + self: Pin<&Self>, + key: StoreKey<'_>, + writer: &mut DropCloserWriteHalf, + offset: u64, + length: Option, + ) -> Result<(), Error> { + self.get_part_identity(key, writer, offset, length).await + } + + fn inner_store(&self, _key: Option) -> &dyn StoreDriver { + // Representation-changing store => terminal. + self + } + + fn as_any(&self) -> &(dyn core::any::Any + Sync + Send + 'static) { + self + } + + fn as_any_arc(self: Arc) -> Arc { + self + } + + fn register_remove_callback(self: Arc, callback: RemoveCallback) -> Result<(), Error> { + self.inner_store.register_remove_callback(callback) + } +} + +default_health_status_indicator!(ZstdStore); + +#[cfg(test)] +mod tests { + use std::io::Write; + + use nativelink_util::digest_hasher::DigestHasherFunc; + + use super::{DecodeSink, create_temp_exclusive}; + use crate::cas_utils::is_zero_digest; + + fn unique_temp_path(tag: &str) -> std::path::PathBuf { + // Process- and call-unique name under the system temp dir. `Math`-style + // randomness is unavailable in some sandboxes, so combine the pid with a + // monotonic counter instead of a clock. + use core::sync::atomic::{AtomicU64, Ordering}; + static COUNTER: AtomicU64 = AtomicU64::new(0); + let n = COUNTER.fetch_add(1, Ordering::Relaxed); + std::env::temp_dir().join(format!( + "nativelink-zstd-unit-{tag}-{}-{n}", + std::process::id() + )) + } + + #[test] + fn create_temp_exclusive_rejects_existing_path() { + let path = unique_temp_path("exists"); + let path_str = path.to_str().unwrap(); + let first = create_temp_exclusive(path_str).expect("first exclusive create must succeed"); + drop(first); + let err = create_temp_exclusive(path_str) + .expect_err("second exclusive create on an existing path must fail"); + assert!( + err.to_string().contains("exclusively create"), + "unexpected error: {err}" + ); + drop(std::fs::remove_file(path_str)); + } + + #[cfg(unix)] + #[test] + fn create_temp_exclusive_rejects_symlink() { + let target = unique_temp_path("symlink-target"); + let link = unique_temp_path("symlink-link"); + let link_str = link.to_str().unwrap(); + // A dangling symlink at the path: O_EXCL must refuse to create through it. + std::os::unix::fs::symlink(&target, &link).expect("symlink must be created"); + let err = create_temp_exclusive(link_str) + .expect_err("exclusive create through a symlink must fail"); + assert!( + err.to_string().contains("exclusively create"), + "unexpected error: {err}" + ); + // The symlink target must never have been created (no follow). + assert!(!target.exists(), "O_EXCL must not follow the symlink"); + drop(std::fs::remove_file(link_str)); + } + + #[cfg(unix)] + #[test] + fn create_temp_exclusive_sets_owner_only_mode() { + use std::os::unix::fs::PermissionsExt; + let path = unique_temp_path("mode"); + let path_str = path.to_str().unwrap(); + let file = create_temp_exclusive(path_str).expect("exclusive create must succeed"); + let mode = file.metadata().unwrap().permissions().mode() & 0o777; + assert_eq!( + mode, 0o600, + "staging file must be created 0o600, got {mode:o}" + ); + drop(file); + drop(std::fs::remove_file(path_str)); + } + + #[test] + fn decode_sink_rejects_output_past_max_and_never_overcounts() { + // Cap at 4 bytes; a write that would exceed it is rejected wholesale and + // must not advance the counter or hash any of the offending bytes. + let mut sink = DecodeSink::new(DigestHasherFunc::Sha256, 4, None); + assert_eq!( + sink.write(b"abcd").unwrap(), + 4, + "exact-fit write is accepted" + ); + assert_eq!(sink.decoded_len, 4); + let err = sink + .write(b"e") + .expect_err("a byte past the cap must be rejected at the sink"); + assert_eq!(err.kind(), std::io::ErrorKind::InvalidData); + assert_eq!(sink.decoded_len, 4, "rejected bytes must not be counted"); + } + + #[test] + fn decode_sink_zero_cap_rejects_first_byte() { + // The zero-digest configuration: any decoded byte is rejected. + let mut sink = DecodeSink::new(DigestHasherFunc::Sha256, 0, None); + assert!( + sink.write(b"x").is_err(), + "a zero-capacity sink must reject any output" + ); + assert_eq!(sink.decoded_len, 0); + // An empty write is a no-op success (decoders may flush zero bytes). + assert_eq!(sink.write(b"").unwrap(), 0); + } + + #[test] + fn zero_byte_digests_are_recognized() { + // Guards the invariant the zero-digest fast paths rely on. + for digest in crate::cas_utils::ZERO_BYTE_DIGESTS { + assert!(is_zero_digest(digest)); + } + } +} diff --git a/nativelink-store/tests/fast_slow_store_test.rs b/nativelink-store/tests/fast_slow_store_test.rs index 769c489e6..b6ee7ee5a 100644 --- a/nativelink-store/tests/fast_slow_store_test.rs +++ b/nativelink-store/tests/fast_slow_store_test.rs @@ -33,7 +33,7 @@ use nativelink_util::buf_channel::{ use nativelink_util::common::DigestInfo; use nativelink_util::health_utils::{HealthStatusIndicator, default_health_status_indicator}; use nativelink_util::store_trait::{ - RemoveItemCallback, Store, StoreDriver, StoreKey, StoreLike, UploadSizeInfo, + RemoveCallback, Store, StoreDriver, StoreKey, StoreLike, UploadSizeInfo, }; use pretty_assertions::assert_eq; use rand::rngs::SmallRng; @@ -323,7 +323,7 @@ async fn drop_on_eof_completes_store_futures() -> Result<(), Error> { fn register_remove_callback( self: Arc, - _callback: Arc, + _callback: RemoveCallback, ) -> Result<(), Error> { Ok(()) } @@ -654,7 +654,7 @@ fn make_stores_with_lazy_slow() -> (Store, Store, Store) { fn register_remove_callback( self: Arc, - _callback: Arc, + _callback: RemoveCallback, ) -> Result<(), Error> { Ok(()) } @@ -794,10 +794,7 @@ impl StoreDriver for InstrumentedSlowStore { self } - fn register_remove_callback( - self: Arc, - _callback: Arc, - ) -> Result<(), Error> { + fn register_remove_callback(self: Arc, _callback: RemoveCallback) -> Result<(), Error> { Ok(()) } } @@ -1050,7 +1047,7 @@ async fn has_sees_in_flight_slow_writes() -> Result<(), Error> { fn register_remove_callback( self: Arc, - _callback: Arc, + _callback: RemoveCallback, ) -> Result<(), Error> { Ok(()) } @@ -1210,7 +1207,7 @@ async fn has_does_not_consult_fast_store_when_slow_store_hits() -> Result<(), Er fn register_remove_callback( self: Arc, - _callback: Arc, + _callback: RemoveCallback, ) -> Result<(), Error> { Ok(()) } @@ -1316,10 +1313,7 @@ impl StoreDriver for GatedSlowStore2 { fn as_any_arc(self: Arc) -> Arc { self } - fn register_remove_callback( - self: Arc, - _callback: Arc, - ) -> Result<(), Error> { + fn register_remove_callback(self: Arc, _callback: RemoveCallback) -> Result<(), Error> { Ok(()) } } @@ -1476,10 +1470,7 @@ impl StoreDriver for MapBackedSlow { fn as_any_arc(self: Arc) -> Arc { self } - fn register_remove_callback( - self: Arc, - _cb: Arc, - ) -> Result<(), Error> { + fn register_remove_callback(self: Arc, _cb: RemoveCallback) -> Result<(), Error> { Ok(()) } } @@ -1658,10 +1649,7 @@ impl StoreDriver for CountingSlowStore { fn as_any_arc(self: Arc) -> Arc { self } - fn register_remove_callback( - self: Arc, - _callback: Arc, - ) -> Result<(), Error> { + fn register_remove_callback(self: Arc, _callback: RemoveCallback) -> Result<(), Error> { Ok(()) } } @@ -1922,10 +1910,7 @@ impl StoreDriver for StaleFastStore { self } - fn register_remove_callback( - self: Arc, - _callback: Arc, - ) -> Result<(), Error> { + fn register_remove_callback(self: Arc, _callback: RemoveCallback) -> Result<(), Error> { Ok(()) } } diff --git a/nativelink-store/tests/filesystem_store_test.rs b/nativelink-store/tests/filesystem_store_test.rs index 139a40b44..17da82e59 100644 --- a/nativelink-store/tests/filesystem_store_test.rs +++ b/nativelink-store/tests/filesystem_store_test.rs @@ -271,6 +271,33 @@ const VALUE1: &str = "0123456789"; const VALUE2: &str = "9876543210"; const STRING_NAME: &str = "String_Filename"; +#[nativelink_test] +async fn evict_page_cache_defaults_off_and_round_trips_when_on() -> Result<(), Error> { + assert!( + !FilesystemSpec::default().evict_page_cache, + "evict_page_cache should default to off" + ); + + let digest = DigestInfo::try_new(HASH1, VALUE1.len())?; + let store = Store::new( + FilesystemStore::::new(&FilesystemSpec { + content_path: make_temp_path("content_path"), + temp_path: make_temp_path("temp_path"), + eviction_policy: None, + block_size: 1, + evict_page_cache: true, + ..Default::default() + }) + .await?, + ); + store.update_oneshot(digest, VALUE1.into()).await?; + let content = store + .get_part_unchunked(StoreKey::Digest(digest), 0, None) + .await?; + assert_eq!(content, VALUE1.as_bytes()); + Ok(()) +} + #[nativelink_test] async fn valid_results_after_shutdown_test() -> Result<(), Error> { let digest = DigestInfo::try_new(HASH1, VALUE1.len())?; @@ -1663,6 +1690,66 @@ async fn executable_hardlink_source_created_once_and_readonly() -> Result<(), Er Ok(()) } +/// A store whose `.exec` directory cannot be cleared must still start and +/// serve ordinary CAS reads. It must not, however, trust an executable variant +/// left behind by an earlier process: that file may be stale or torn after an +/// unclean shutdown, so the executable hardlink fast path stays disabled until +/// the directory can be wiped on a later writable startup. +#[cfg(target_family = "unix")] +#[nativelink_test] +async fn read_only_executable_dir_preserves_cas_reads_without_reusing_variants() -> Result<(), Error> +{ + use std::os::unix::fs::PermissionsExt; + + struct RestoreWritable(String); + + impl Drop for RestoreWritable { + fn drop(&mut self) { + use std::os::unix::fs::PermissionsExt; + + std::fs::set_permissions(&self.0, std::fs::Permissions::from_mode(0o755)) + .expect("restore executable digest directory permissions"); + } + } + + let content_path = make_temp_path("content_path_read_only_exec"); + let temp_path = make_temp_path("temp_path_read_only_exec"); + let digest = DigestInfo::try_new(HASH1, VALUE1.len())?; + let spec = FilesystemSpec { + content_path: content_path.clone(), + temp_path, + ..Default::default() + }; + + { + let store = FilesystemStore::::new(&spec).await?; + store.update_oneshot(digest, VALUE1.into()).await?; + store.get_executable_hardlink_source(&digest).await?; + } + + let executable_digest_dir = format!("{content_path}.exec/{DIGEST_FOLDER}"); + fs::set_permissions( + &executable_digest_dir, + std::fs::Permissions::from_mode(0o555), + ) + .await?; + let _restore_permissions = RestoreWritable(executable_digest_dir); + + let store = FilesystemStore::::new(&spec).await?; + assert_eq!( + store.get_part_unchunked(digest, 0, None).await?, + VALUE1.as_bytes(), + "read-only executable cleanup must not prevent ordinary CAS reads" + ); + let err = store + .get_executable_hardlink_source(&digest) + .await + .expect_err("a surviving executable variant must not be trusted"); + assert_eq!(err.code, Code::FailedPrecondition); + + Ok(()) +} + /// Regression test for #2474: the `.exec` variant directory was never /// registered in `evicting_map`, so it was invisible to `max_bytes` and only /// ever cleared by the startup `remove_dir_all` — growing without bound at @@ -1982,3 +2069,74 @@ async fn unref_does_not_orphan_content_file_when_temp_dir_missing() -> Result<() Ok(()) } + +// Exercises the macOS flush coalescer's round machinery under a burst of +// concurrent uploads (on other platforms this is a plain concurrency smoke +// test). Two hundred small, independently durable blobs are used because each +// upload would otherwise issue its own multi-millisecond `F_FULLFSYNC` on +// macOS. The data volume is deliberately small so file I/O does not hide the +// synchronization behavior: every blob must complete and survive restart, +// while the device-wide barrier count must remain far below the blob count. +#[nativelink_test(flavor = "multi_thread", worker_threads = 4)] +async fn concurrent_upload_burst_round_trip_test() -> Result<(), Error> { + const NUM_BLOBS: u64 = 200; + let content_path = make_temp_path("content_path_burst"); + let temp_path = make_temp_path("temp_path_burst"); + let spec = FilesystemSpec { + content_path: content_path.clone(), + temp_path: temp_path.clone(), + eviction_policy: None, + block_size: 1, + ..Default::default() + }; + let make_digest = |i: u64| { + let mut hash = [0u8; 32]; + hash[0] = 0xbb; + hash[1..9].copy_from_slice(&i.to_le_bytes()); + DigestInfo::new(hash, 25) + }; + { + let filesystem_store = FilesystemStore::::new(&spec).await?; + let store = Store::new(filesystem_store.clone()); + let mut handles = Vec::new(); + for i in 0..NUM_BLOBS { + let store = store.clone(); + handles.push(tokio::spawn(async move { + let content = format!("burst-content-{i:011}"); + store + .update_oneshot(make_digest(i), content.into_bytes().into()) + .await + })); + } + for handle in handles { + handle.await.expect("upload task panicked")?; + } + #[cfg(target_os = "macos")] + { + let full_flush_count = filesystem_store + .full_flush_count_for_test() + .expect("writable test store must have a flush coalescer"); + assert!( + full_flush_count > 0 && full_flush_count < NUM_BLOBS / 2, + "{NUM_BLOBS} concurrent uploads should share device-wide barriers; observed {full_flush_count} F_FULLFSYNC rounds" + ); + } + for i in 0..NUM_BLOBS { + assert_eq!( + store.get_part_unchunked(make_digest(i), 0, None).await?, + format!("burst-content-{i:011}").as_bytes(), + "round trip failed for blob {i}" + ); + } + } + // A fresh store over the same paths must find every published blob. + let store = Store::new(FilesystemStore::::new(&spec).await?); + for i in 0..NUM_BLOBS { + assert_eq!( + store.has(make_digest(i)).await?, + Some(25), + "blob {i} missing after restart" + ); + } + Ok(()) +} diff --git a/nativelink-store/tests/gcs_store_test.rs b/nativelink-store/tests/gcs_store_test.rs index 34f9cba4d..93fae5c6f 100644 --- a/nativelink-store/tests/gcs_store_test.rs +++ b/nativelink-store/tests/gcs_store_test.rs @@ -18,7 +18,7 @@ use std::sync::Arc; use bytes::{BufMut, Bytes, BytesMut}; use mock_instant::thread_local::MockClock; -use nativelink_config::stores::{CommonObjectSpec, ExperimentalGcsSpec}; +use nativelink_config::stores::{CommonObjectSpec, ErrorCode, ExperimentalGcsSpec, Retry}; use nativelink_error::{Code, Error, make_err}; use nativelink_macro::nativelink_test; use nativelink_store::cas_utils::ZERO_BYTE_DIGESTS; @@ -181,6 +181,90 @@ async fn get_part_handles_not_found_error() -> Result<(), Error> { Ok(()) } +#[nativelink_test] +async fn get_part_not_found_not_retried_by_default() -> Result<(), Error> { + // Create mock GCS operations without adding any object, so reads + // yield NotFound. + let mock_ops = Arc::new(MockGcsOperations::new()); + let store = create_test_store_with_retry( + mock_ops.clone(), + Retry { + max_retries: 2, + delay: 0.001, + jitter: 0.0, + ..Default::default() + }, + ) + .await?; + + let digest = DigestInfo::try_new(VALID_HASH1, 100)?; + let store_key: StoreKey = to_store_key(digest); + let (mut tx, _rx) = make_buf_channel_pair(); + + let store_clone = store.clone(); + let get_part_fut = nativelink_util::spawn!("get_part_task", async move { + store_clone.get_part(store_key, &mut tx, 0, None).await + }); + + let err = get_part_fut.await?.unwrap_err(); + assert_eq!(err.code, Code::NotFound, "Expected NotFound error"); + + // Even with a retry budget configured, NotFound must not consume it + // unless explicitly opted in via `retry_on_errors`. + let call_counts = mock_ops.get_call_counts(); + assert_eq!( + call_counts.read_calls.load(Ordering::Relaxed), + 1, + "read_object_content should not be retried on NotFound by default" + ); + + Ok(()) +} + +#[nativelink_test] +async fn get_part_retries_not_found_when_opted_in() -> Result<(), Error> { + // Create mock GCS operations without adding any object, so reads + // yield NotFound. + let mock_ops = Arc::new(MockGcsOperations::new()); + let store = create_test_store_with_retry( + mock_ops.clone(), + Retry { + max_retries: 2, + delay: 0.001, + jitter: 0.0, + retry_on_errors: Some(vec![ErrorCode::NotFound]), + }, + ) + .await?; + + let digest = DigestInfo::try_new(VALID_HASH1, 100)?; + let store_key: StoreKey = to_store_key(digest); + let (mut tx, _rx) = make_buf_channel_pair(); + + let store_clone = store.clone(); + let get_part_fut = nativelink_util::spawn!("get_part_task", async move { + store_clone.get_part(store_key, &mut tx, 0, None).await + }); + + let err = get_part_fut.await?.unwrap_err(); + assert_eq!( + err.code, + Code::NotFound, + "Expected NotFound error after retries are exhausted" + ); + + // With NotFound in `retry_on_errors`, the read should be attempted + // once plus `max_retries` more times before giving up. + let call_counts = mock_ops.get_call_counts(); + assert_eq!( + call_counts.read_calls.load(Ordering::Relaxed), + 3, + "read_object_content should be retried on NotFound when opted in" + ); + + Ok(()) +} + #[nativelink_test] async fn has_with_results_test() -> Result<(), Error> { // Create mock GCS operations @@ -528,9 +612,7 @@ async fn test_expired_object() -> Result<(), Error> { .await; // Mock the now function to return a time before expiration - MockClock::set_time(Duration::from_secs( - base_timestamp as u64 + expiration_seconds as u64 - 1, - )); + MockClock::set_time(Duration::from_secs(base_timestamp + expiration_seconds - 1)); // Check that the object exists (it's not expired yet) let result = store.has(store_key.clone()).await?; @@ -541,9 +623,7 @@ async fn test_expired_object() -> Result<(), Error> { ); // Mock the now function to return a time after expiration - MockClock::set_time(Duration::from_secs( - base_timestamp as u64 + expiration_seconds as u64 + 1, - )); + MockClock::set_time(Duration::from_secs(base_timestamp + expiration_seconds + 1)); // Object should now be considered expired let result = store.has(store_key).await?; @@ -648,7 +728,7 @@ async fn test_null_object_metadata() -> Result<(), Error> { let timestamp = std::time::SystemTime::now() .duration_since(std::time::UNIX_EPOCH) .unwrap_or_default() - .as_secs() as i64; + .as_secs(); let metadata = nativelink_store::gcs_client::types::GcsObject { name: object_path.path.clone(), @@ -705,10 +785,30 @@ async fn create_test_store( ) } +// Helper function to create a test GCS store with a custom retry config +async fn create_test_store_with_retry( + ops: Arc, + retry: Retry, +) -> Result MockInstantWrapped>>, Error> { + GcsStore::new_with_ops( + &ExperimentalGcsSpec { + bucket: BUCKET_NAME.to_string(), + common: CommonObjectSpec { + key_prefix: Some(KEY_PREFIX.to_string()), + retry, + ..Default::default() + }, + ..Default::default() + }, + ops, + MockInstantWrapped::default, + ) +} + // Helper function to create a test GCS store with expiration async fn create_test_store_with_expiration( ops: Arc, - expiration_seconds: i64, + expiration_seconds: u64, ) -> Result MockInstantWrapped>>, Error> { GcsStore::new_with_ops( &ExperimentalGcsSpec { @@ -771,3 +871,148 @@ async fn check_health_failed_on_object_exists_error() -> Result<(), Error> { other => panic!("expected HealthStatus::Failed, got {other:?}"), } } + +#[nativelink_test] +async fn get_part_ignores_empty_stream_chunks() -> Result<(), Error> { + use futures::stream::{self, Stream, StreamExt}; + use nativelink_store::gcs_client::types::GcsObject; + use nativelink_util::buf_channel::DropCloserReadHalf; + + /// Wraps `MockGcsOperations`, re-chunking every content read so the + /// stream interleaves zero-length chunks the way an HTTP/2 peer that + /// emits empty DATA frames does. + #[derive(Debug)] + struct EmptyChunkOps(Arc); + + impl GcsOperations for EmptyChunkOps { + async fn read_object_metadata( + &self, + object: &ObjectPath, + ) -> Result, Error> { + self.0.read_object_metadata(object).await + } + + async fn read_object_content( + &self, + object_path: &ObjectPath, + start: u64, + end: Option, + ) -> Result> + Send + Unpin>, Error> { + let mut inner = self.0.read_object_content(object_path, start, end).await?; + let mut content = BytesMut::new(); + while let Some(chunk) = inner.next().await { + content.put(chunk?); + } + let content = content.freeze(); + let mid = content.len() / 2; + Ok(Box::new(stream::iter(vec![ + Ok(Bytes::new()), + Ok(content.slice(..mid)), + Ok(Bytes::new()), + Ok(content.slice(mid..)), + Ok(Bytes::new()), + ]))) + } + + async fn write_object( + &self, + object_path: &ObjectPath, + content: Vec, + ) -> Result<(), Error> { + self.0.write_object(object_path, content).await + } + + async fn start_resumable_write(&self, object_path: &ObjectPath) -> Result { + self.0.start_resumable_write(object_path).await + } + + async fn upload_chunk( + &self, + upload_url: &str, + object_path: &ObjectPath, + data: Bytes, + offset: u64, + end_offset: u64, + total_size: Option, + ) -> Result<(), Error> { + self.0 + .upload_chunk( + upload_url, + object_path, + data, + offset, + end_offset, + total_size, + ) + .await + } + + async fn upload_from_reader( + &self, + object_path: &ObjectPath, + reader: &mut DropCloserReadHalf, + upload_id: &str, + max_size: u64, + ) -> Result<(), Error> { + self.0 + .upload_from_reader(object_path, reader, upload_id, max_size) + .await + } + + async fn object_exists(&self, object_path: &ObjectPath) -> Result { + self.0.object_exists(object_path).await + } + } + + let mock_ops = Arc::new(MockGcsOperations::new()); + let digest = DigestInfo::try_new(VALID_HASH1, 11)?; // "hello world" length + let store_key: StoreKey = to_store_key(digest); + let object_path = create_object_path(&store_key); + mock_ops + .add_object(&object_path, b"hello world".to_vec()) + .await; + + let store = GcsStore::new_with_ops( + &ExperimentalGcsSpec { + bucket: BUCKET_NAME.to_string(), + common: CommonObjectSpec { + key_prefix: Some(KEY_PREFIX.to_string()), + ..Default::default() + }, + ..Default::default() + }, + Arc::new(EmptyChunkOps(mock_ops)), + MockInstantWrapped::default, + )?; + + let (mut tx, mut rx) = make_buf_channel_pair(); + let store_clone = store.clone(); + let store_key_clone = store_key.clone(); + + let handle = nativelink_util::spawn!("get_part_task", async move { + store_clone + .get_part(store_key_clone, &mut tx, 0, None) + .await + }); + + let received_data = + match tokio::time::timeout(Duration::from_secs(5), rx.consume(Some(100))).await { + Ok(result) => result?, + Err(_) => { + return Err(make_err!( + Code::DeadlineExceeded, + "Timeout waiting for data" + )); + } + }; + + assert_eq!( + received_data.as_ref(), + b"hello world", + "Received data should match original despite empty stream chunks" + ); + + handle.await??; + + Ok(()) +} diff --git a/nativelink-store/tests/grpc_read_batching_test.rs b/nativelink-store/tests/grpc_read_batching_test.rs new file mode 100644 index 000000000..a31e63feb --- /dev/null +++ b/nativelink-store/tests/grpc_read_batching_test.rs @@ -0,0 +1,574 @@ +// Copyright 2026 The NativeLink Authors. All rights reserved. +// +// Licensed under the Functional Source License, Version 1.1, Apache 2.0 Future License (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// See LICENSE file for details +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use core::future::Future; +use core::pin::Pin; +use core::task::{Context as TaskContext, Waker}; +use std::collections::{HashMap, HashSet}; +use std::sync::Arc; + +use async_lock::Mutex; +use bytes::Bytes; +use futures::Stream; +use nativelink_config::stores::{GrpcEndpoint, GrpcReadBatchingConfig, GrpcSpec, Retry, StoreType}; +use nativelink_error::{Code, Error}; +use nativelink_macro::nativelink_test; +use nativelink_proto::build::bazel::remote::execution::v2::content_addressable_storage_server::{ + ContentAddressableStorage, ContentAddressableStorageServer, +}; +use nativelink_proto::build::bazel::remote::execution::v2::{ + BatchReadBlobsRequest, BatchReadBlobsResponse, BatchUpdateBlobsRequest, + BatchUpdateBlobsResponse, FindMissingBlobsRequest, FindMissingBlobsResponse, GetTreeRequest, + GetTreeResponse, SpliceBlobRequest, SpliceBlobResponse, SplitBlobRequest, SplitBlobResponse, + batch_read_blobs_response, +}; +use nativelink_proto::google::bytestream::byte_stream_server::{ByteStream, ByteStreamServer}; +use nativelink_proto::google::bytestream::{ + QueryWriteStatusRequest, QueryWriteStatusResponse, ReadRequest, ReadResponse, WriteRequest, + WriteResponse, +}; +use nativelink_proto::google::rpc::Status as RpcStatus; +use nativelink_store::grpc_store::GrpcStore; +use nativelink_util::background_spawn; +use nativelink_util::common::DigestInfo; +use nativelink_util::store_trait::StoreLike; +use tonic::transport::Server; +use tonic::transport::server::TcpIncoming; +use tonic::{Request, Response, Status, Streaming}; + +/// A fake CAS server that serves seeded blobs from `BatchReadBlobs` and +/// records every `BatchReadBlobs` request it receives. Digests listed in +/// `not_found_hashes` are reported as `NOT_FOUND` in the per-blob status. +#[derive(Debug, Clone)] +struct FakeCasServer { + blobs: Arc>>, + not_found_hashes: Arc>>, + batch_read_requests: Arc>>, +} + +impl FakeCasServer { + fn new() -> Self { + Self { + blobs: Arc::new(Mutex::new(HashMap::new())), + not_found_hashes: Arc::new(Mutex::new(vec![])), + batch_read_requests: Arc::new(Mutex::new(vec![])), + } + } +} + +type GetTreeStream = Pin> + Send + 'static>>; + +#[tonic::async_trait] +impl ContentAddressableStorage for FakeCasServer { + type GetTreeStream = GetTreeStream; + + #[allow(clippy::unimplemented)] + async fn find_missing_blobs( + &self, + _grpc_request: Request, + ) -> Result, Status> { + unimplemented!(); + } + + #[allow(clippy::unimplemented)] + async fn batch_update_blobs( + &self, + _grpc_request: Request, + ) -> Result, Status> { + unimplemented!(); + } + + async fn batch_read_blobs( + &self, + grpc_request: Request, + ) -> Result, Status> { + let request = grpc_request.into_inner(); + self.batch_read_requests.lock().await.push(request.clone()); + + let blobs = self.blobs.lock().await; + let not_found_hashes = self.not_found_hashes.lock().await; + let mut responses = Vec::with_capacity(request.digests.len()); + // Real servers may dedupe duplicate digests in one request; mimic + // that by returning exactly one entry per unique digest. + let mut seen_hashes = HashSet::new(); + for digest in request.digests { + if !seen_hashes.insert(digest.hash.clone()) { + continue; + } + let (data, status) = if not_found_hashes.contains(&digest.hash) { + ( + Bytes::new(), + RpcStatus { + code: Code::NotFound as i32, + message: format!("Blob {} not found", digest.hash), + details: vec![], + }, + ) + } else if let Some(data) = blobs.get(&digest.hash) { + ( + data.clone(), + RpcStatus { + code: Code::Ok as i32, + message: String::new(), + details: vec![], + }, + ) + } else { + return Err(Status::invalid_argument(format!( + "Blob {} was never seeded in FakeCasServer", + digest.hash + ))); + }; + responses.push(batch_read_blobs_response::Response { + digest: Some(digest), + data, + compressor: 0, + status: Some(status), + }); + } + Ok(Response::new(BatchReadBlobsResponse { responses })) + } + + #[allow(clippy::unimplemented)] + async fn get_tree( + &self, + _grpc_request: Request, + ) -> Result, Status> { + unimplemented!(); + } + + #[allow(clippy::unimplemented)] + async fn split_blob( + &self, + _grpc_request: Request, + ) -> Result, Status> { + unimplemented!(); + } + + #[allow(clippy::unimplemented)] + async fn splice_blob( + &self, + _grpc_request: Request, + ) -> Result, Status> { + unimplemented!(); + } +} + +type ReadStream = Pin> + Send + 'static>>; + +/// A fake `ByteStream` server that serves the same seeded blobs as +/// [`FakeCasServer`] and records every `Read` request it receives. +#[derive(Debug, Clone)] +struct FakeByteStreamServer { + blobs: Arc>>, + read_requests: Arc>>, +} + +impl FakeByteStreamServer { + fn new(blobs: Arc>>) -> Self { + Self { + blobs, + read_requests: Arc::new(Mutex::new(vec![])), + } + } +} + +#[tonic::async_trait] +impl ByteStream for FakeByteStreamServer { + type ReadStream = ReadStream; + + async fn read( + &self, + grpc_request: Request, + ) -> Result, Status> { + let request = grpc_request.into_inner(); + self.read_requests.lock().await.push(request.clone()); + + // Resource names look like `{instance}/blobs/{fn}/{hash}/{size}`. + let mut components = request.resource_name.rsplit('/'); + let _size = components.next(); + let hash = components.next().unwrap_or_default().to_string(); + let data = self + .blobs + .lock() + .await + .get(&hash) + .cloned() + .ok_or_else(|| Status::not_found(format!("Blob {hash} not seeded")))?; + let offset = usize::try_from(request.read_offset) + .map_err(|_| Status::invalid_argument("Bad read_offset"))?; + if offset > data.len() { + return Err(Status::out_of_range("read_offset past end of blob")); + } + let stream: Self::ReadStream = Box::pin(futures::stream::iter(vec![Ok(ReadResponse { + data: data.slice(offset..), + })])); + Ok(Response::new(stream)) + } + + #[allow(clippy::unimplemented)] + async fn write( + &self, + _grpc_request: Request>, + ) -> Result, Status> { + unimplemented!(); + } + + #[allow(clippy::unimplemented)] + async fn query_write_status( + &self, + _grpc_request: Request, + ) -> Result, Status> { + unimplemented!(); + } +} + +struct TestFixture { + cas_server: FakeCasServer, + bytestream_server: FakeByteStreamServer, + store: Arc, +} + +async fn make_fixture(read_batching: Option) -> Result { + let cas_server = FakeCasServer::new(); + let bytestream_server = FakeByteStreamServer::new(cas_server.blobs.clone()); + let listener = TcpIncoming::bind("127.0.0.1:0".parse().unwrap()).unwrap(); + let port = listener.local_addr().unwrap().port(); + + let cas_service = ContentAddressableStorageServer::new(cas_server.clone()); + let bytestream_service = ByteStreamServer::new(bytestream_server.clone()); + background_spawn!("grpc_read_batching_test_server", async move { + Server::builder() + .add_service(cas_service) + .add_service(bytestream_service) + .serve_with_incoming(listener) + .await + .unwrap(); + }); + + let spec = GrpcSpec { + instance_name: String::new(), + endpoints: vec![GrpcEndpoint { + address: format!("http://localhost:{port}"), + tls_config: None, + concurrency_limit: None, + connect_timeout_s: 0, + tcp_keepalive_s: 0, + http2_keepalive_interval_s: 0, + http2_keepalive_timeout_s: 0, + }], + store_type: StoreType::Cas, + retry: Retry::default(), + max_concurrent_requests: 0, + connections_per_endpoint: 0, + rpc_timeout_s: 0, + use_legacy_resource_names: false, + headers: HashMap::new(), + forward_headers: vec![], + experimental_read_batching: read_batching, + experimental_remote_cache_compression: false, + }; + let store = GrpcStore::new(&spec).await?; + Ok(TestFixture { + cas_server, + bytestream_server, + store, + }) +} + +const fn batching_config() -> GrpcReadBatchingConfig { + GrpcReadBatchingConfig { + max_blob_size_bytes: 128 * 1024, + max_batch_bytes: 3 * 1024 * 1024, + dispatch_slots: 1, + max_queued_bytes: 32 * 1024 * 1024, + } +} + +/// Creates a unique digest and 64-byte content for blob index `i`. +fn make_blob(i: usize) -> (DigestInfo, Bytes) { + let hash = format!("{:064x}", i + 1); + let content = Bytes::from(format!("{i:0>64}")); + let digest = DigestInfo::try_new(&hash, content.len()).unwrap(); + (digest, content) +} + +async fn seed_blobs(fixture: &TestFixture, blobs: &[(DigestInfo, Bytes)]) { + let mut map = fixture.cas_server.blobs.lock().await; + for (digest, content) in blobs { + map.insert(digest.packed_hash().to_string(), content.clone()); + } +} + +// Many concurrent small reads must be coalesced into BatchReadBlobs RPCs +// and never touch the ByteStream Read path. +#[nativelink_test] +async fn small_blob_reads_are_batched() -> Result<(), Error> { + const NUM_BLOBS: usize = 200; + + let fixture = make_fixture(Some(batching_config())).await?; + let mut blobs = Vec::with_capacity(NUM_BLOBS); + for i in 0..NUM_BLOBS { + blobs.push(make_blob(i)); + } + seed_blobs(&fixture, &blobs).await; + + let mut read_futures = Vec::with_capacity(NUM_BLOBS); + for (digest, _) in &blobs { + read_futures.push(fixture.store.get_part_unchunked(*digest, 0, None)); + } + let results = futures::future::join_all(read_futures).await; + for (result, (_, expected_content)) in results.into_iter().zip(&blobs) { + assert_eq!(&result?, expected_content); + } + + let batch_read_requests = fixture.cas_server.batch_read_requests.lock().await; + assert!( + !batch_read_requests.is_empty(), + "Expected BatchReadBlobs to be used" + ); + let total_batched_digests: usize = batch_read_requests + .iter() + .map(|request| request.digests.len()) + .sum(); + assert_eq!(total_batched_digests, NUM_BLOBS); + assert!( + batch_read_requests.len() < NUM_BLOBS / 2, + "Expected reads to be coalesced, got {} BatchReadBlobs calls", + batch_read_requests.len() + ); + assert_eq!( + fixture.bytestream_server.read_requests.lock().await.len(), + 0, + "ByteStream Read must not be used for batched reads" + ); + Ok(()) +} + +// Blobs above max_blob_size_bytes must use the ByteStream Read path. +#[nativelink_test] +async fn large_blob_uses_stream_path() -> Result<(), Error> { + let mut config = batching_config(); + config.max_blob_size_bytes = 1024; + let fixture = make_fixture(Some(config)).await?; + + let hash = format!("{:064x}", 42_u32); + let content = Bytes::from(vec![42_u8; 2048]); + let digest = DigestInfo::try_new(&hash, content.len()).unwrap(); + seed_blobs(&fixture, &[(digest, content.clone())]).await; + + let data = fixture.store.get_part_unchunked(digest, 0, None).await?; + assert_eq!(data, content); + + assert_eq!( + fixture.bytestream_server.read_requests.lock().await.len(), + 1 + ); + assert_eq!(fixture.cas_server.batch_read_requests.lock().await.len(), 0); + Ok(()) +} + +// Partial reads (offset != 0) must bypass batching. +#[nativelink_test] +async fn partial_read_bypasses_batching() -> Result<(), Error> { + let fixture = make_fixture(Some(batching_config())).await?; + let (digest, content) = make_blob(7); + seed_blobs(&fixture, &[(digest, content.clone())]).await; + + let data = fixture.store.get_part_unchunked(digest, 1, None).await?; + assert_eq!(data, content.slice(1..)); + + let read_requests = fixture.bytestream_server.read_requests.lock().await; + assert_eq!(read_requests.len(), 1); + assert_eq!(read_requests[0].read_offset, 1); + assert_eq!(fixture.cas_server.batch_read_requests.lock().await.len(), 0); + Ok(()) +} + +// A NOT_FOUND for one digest in a batch must fail only that read; the +// other reads in the same batch must succeed. +#[nativelink_test] +async fn batch_item_failure_is_isolated() -> Result<(), Error> { + let fixture = make_fixture(Some(batching_config())).await?; + let (good_digest1, good_content1) = make_blob(1); + let (bad_digest, _) = make_blob(2); + let (good_digest2, good_content2) = make_blob(3); + seed_blobs( + &fixture, + &[ + (good_digest1, good_content1.clone()), + (good_digest2, good_content2.clone()), + ], + ) + .await; + fixture + .cas_server + .not_found_hashes + .lock() + .await + .push(bad_digest.packed_hash().to_string()); + + // On the single-threaded test runtime all three reads enqueue before + // the detached dispatcher task gets to run, so they coalesce. + let (good_result1, bad_result, good_result2) = futures::join!( + fixture.store.get_part_unchunked(good_digest1, 0, None), + fixture.store.get_part_unchunked(bad_digest, 0, None), + fixture.store.get_part_unchunked(good_digest2, 0, None), + ); + assert_eq!(good_result1?, good_content1); + assert_eq!(good_result2?, good_content2); + let bad_err = bad_result.expect_err("Expected NOT_FOUND digest to fail"); + assert_eq!(bad_err.code, Code::NotFound, "Got error: {bad_err:?}"); + + // Confirm that the failed digest actually shared a batch with another + // read (rather than each read getting its own RPC). + let batch_read_requests = fixture.cas_server.batch_read_requests.lock().await; + let bad_hash = bad_digest.packed_hash().to_string(); + assert!( + batch_read_requests + .iter() + .any(|request| request.digests.len() > 1 + && request.digests.iter().any(|d| d.hash == bad_hash)), + "Expected the failing digest to share a batch with other reads: {batch_read_requests:?}" + ); + assert_eq!( + fixture.bytestream_server.read_requests.lock().await.len(), + 0 + ); + Ok(()) +} + +// With batching unconfigured the store must behave exactly as before and +// never call BatchReadBlobs. +#[nativelink_test] +async fn disabled_batching_uses_stream_path() -> Result<(), Error> { + let fixture = make_fixture(None).await?; + let (digest, content) = make_blob(11); + seed_blobs(&fixture, &[(digest, content.clone())]).await; + + let data = fixture.store.get_part_unchunked(digest, 0, None).await?; + assert_eq!(data, content); + + assert_eq!( + fixture.bytestream_server.read_requests.lock().await.len(), + 1 + ); + assert_eq!(fixture.cas_server.batch_read_requests.lock().await.len(), 0); + Ok(()) +} + +// Cancelling one read mid-flight must not abort the shared batch RPC nor +// strand the other queued reads. +#[nativelink_test] +async fn cancelled_read_does_not_affect_batch_mates() -> Result<(), Error> { + let fixture = make_fixture(Some(batching_config())).await?; + let (cancelled_digest, cancelled_content) = make_blob(1); + let (digest_b, content_b) = make_blob(2); + let (digest_c, content_c) = make_blob(3); + seed_blobs( + &fixture, + &[ + (cancelled_digest, cancelled_content), + (digest_b, content_b.clone()), + (digest_c, content_c.clone()), + ], + ) + .await; + + // Poll one read just far enough to enqueue it, then drop it to + // simulate cancellation before its batch has been dispatched. + let waker = Waker::noop(); + let mut task_context = TaskContext::from_waker(waker); + let mut cancelled_read = Box::pin(fixture.store.get_part_unchunked(cancelled_digest, 0, None)); + assert!(cancelled_read.as_mut().poll(&mut task_context).is_pending()); + drop(cancelled_read); + + let (result_b, result_c) = futures::join!( + fixture.store.get_part_unchunked(digest_b, 0, None), + fixture.store.get_part_unchunked(digest_c, 0, None), + ); + assert_eq!(result_b?, content_b); + assert_eq!(result_c?, content_c); + assert_eq!( + fixture.bytestream_server.read_requests.lock().await.len(), + 0 + ); + Ok(()) +} + +// Concurrent reads of the same digest must all succeed even though the +// digest is requested only once per batch and the server responds with a +// single entry for it. +#[nativelink_test] +async fn duplicate_digest_reads_all_succeed() -> Result<(), Error> { + let fixture = make_fixture(Some(batching_config())).await?; + let (digest, content) = make_blob(5); + seed_blobs(&fixture, &[(digest, content.clone())]).await; + + let (result_a, result_b) = futures::join!( + fixture.store.get_part_unchunked(digest, 0, None), + fixture.store.get_part_unchunked(digest, 0, None), + ); + assert_eq!(result_a?, content); + assert_eq!(result_b?, content); + + // The client must dedupe: each request may carry the digest only once. + let batch_read_requests = fixture.cas_server.batch_read_requests.lock().await; + assert!(!batch_read_requests.is_empty()); + for request in batch_read_requests.iter() { + assert_eq!( + request.digests.len(), + 1, + "Expected deduped digests: {request:?}" + ); + } + assert_eq!( + fixture.bytestream_server.read_requests.lock().await.len(), + 0 + ); + Ok(()) +} + +// Batched reads share one upstream RPC, so combining them with per-client +// forwarded headers must be rejected at construction. +#[nativelink_test] +async fn forward_headers_with_batching_rejected() -> Result<(), Error> { + let spec = GrpcSpec { + instance_name: String::new(), + endpoints: vec![GrpcEndpoint { + address: "http://foobar".to_string(), + tls_config: None, + concurrency_limit: None, + connect_timeout_s: 0, + tcp_keepalive_s: 0, + http2_keepalive_interval_s: 0, + http2_keepalive_timeout_s: 0, + }], + store_type: StoreType::Cas, + retry: Retry::default(), + max_concurrent_requests: 0, + connections_per_endpoint: 0, + rpc_timeout_s: 0, + use_legacy_resource_names: false, + headers: HashMap::new(), + forward_headers: vec!["authorization".to_string()], + experimental_read_batching: Some(batching_config()), + experimental_remote_cache_compression: false, + }; + let err = GrpcStore::new(&spec) + .await + .expect_err("Expected construction to fail"); + assert_eq!(err.code, Code::InvalidArgument, "Got error: {err:?}"); + Ok(()) +} diff --git a/nativelink-store/tests/grpc_store_test.rs b/nativelink-store/tests/grpc_store_test.rs index 858d5a653..9e735a9eb 100644 --- a/nativelink-store/tests/grpc_store_test.rs +++ b/nativelink-store/tests/grpc_store_test.rs @@ -61,6 +61,8 @@ fn test_spec>(endpoint: T, use_legacy_resource_names: bool) -> G use_legacy_resource_names, headers: HashMap::new(), forward_headers: vec![], + experimental_read_batching: None, + experimental_remote_cache_compression: false, } } @@ -162,7 +164,7 @@ impl ByteStream for FakeStreamServer { Some(Ok(write_request)) => write_request, }; info!(?write_request, "write request"); - let committed_size = write_request.data.len() as i64; + let committed_size = write_request.data.len().try_into().unwrap_or(i64::MAX); self.write_requests.lock().await.push(write_request); Ok(Response::new(WriteResponse { committed_size })) } @@ -432,7 +434,7 @@ async fn split_and_splice_blob_forward_to_backend() -> Result<(), Error> { let digest = Digest { hash: VALID_HASH.to_string(), - size_bytes: RAW_INPUT.len() as i64, + size_bytes: RAW_INPUT.len().try_into().unwrap_or(i64::MAX), }; let split_response = store diff --git a/nativelink-store/tests/memory_store_test.rs b/nativelink-store/tests/memory_store_test.rs index 996e94736..069ea59e1 100644 --- a/nativelink-store/tests/memory_store_test.rs +++ b/nativelink-store/tests/memory_store_test.rs @@ -117,7 +117,7 @@ fn store_with_max_bytes(max_bytes: usize) -> Store { })) } -// A write whose exact size is >= max_bytes is skipped (drained, never buffered) +// A write whose exact size is >= `max_bytes` is skipped (drained, never buffered) // rather than materialized-then-evicted, and — crucially — it leaves the rest of // the cache untouched (the old buffer-then-evict path would have evicted the // within-budget entry trying to make room for an unstorable blob). @@ -207,7 +207,7 @@ async fn oversized_skip_fires_remove_callbacks() -> Result<(), Error> { } // `MaxSize` is an upper bound, not the real size, so it must NOT trigger the skip: -// a MaxSize over max_bytes whose actual content fits is still cached. +// a `MaxSize` over `max_bytes` whose actual content fits is still cached. #[nativelink_test] async fn max_size_over_budget_with_small_actual_is_stored() -> Result<(), Error> { const DATA: &[u8] = b"ab"; // 2 bytes, well within the 4-byte budget diff --git a/nativelink-store/tests/redis_store_test.rs b/nativelink-store/tests/redis_store_test.rs index 2bdac314c..e60b42efb 100644 --- a/nativelink-store/tests/redis_store_test.rs +++ b/nativelink-store/tests/redis_store_test.rs @@ -13,13 +13,14 @@ // limitations under the License. use core::ops::RangeBounds; +use core::pin::Pin; use core::time::Duration; use std::collections::HashMap; use std::sync::Arc; use bytes::{Bytes, BytesMut}; use futures::TryStreamExt; -use nativelink_config::stores::{RedisMode, RedisSpec}; +use nativelink_config::stores::{ExistenceCacheSpec, RedisMode, RedisSpec, StoreSpec}; use nativelink_error::{Code, Error, ErrorContext, ResultExt, make_err}; use nativelink_macro::nativelink_test; use nativelink_redis_tester::{ @@ -28,22 +29,24 @@ use nativelink_redis_tester::{ make_fake_redis_with_responses, }; use nativelink_store::cas_utils::ZERO_BYTE_DIGESTS; +use nativelink_store::existence_cache_store::ExistenceCacheStore; use nativelink_store::redis_store::{ ClusterRedisManager, DEFAULT_MAX_CHUNK_UPLOADS_PER_UPDATE, DEFAULT_MAX_COUNT_PER_CURSOR, - LUA_VERSION_SET_SCRIPT, RedisStore, RedisSubscriptionManager, + LUA_VERSION_SET_SCRIPT, RedisStore, RedisSubscriptionManager, decode_key, }; use nativelink_util::buf_channel::make_buf_channel_pair; use nativelink_util::common::DigestInfo; use nativelink_util::health_utils::HealthStatus; use nativelink_util::store_trait::{ - FalseValue, SchedulerCurrentVersionProvider, SchedulerIndexProvider, SchedulerStore, - SchedulerStoreDataProvider, SchedulerStoreDecodeTo, SchedulerStoreKeyProvider, - SchedulerSubscription, SchedulerSubscriptionManager, StoreKey, StoreLike, TrueValue, - UploadSizeInfo, + FalseValue, RemoveItemCallback, SchedulerCurrentVersionProvider, SchedulerIndexProvider, + SchedulerStore, SchedulerStoreDataProvider, SchedulerStoreDecodeTo, SchedulerStoreKeyProvider, + SchedulerSubscription, SchedulerSubscriptionManager, Store, StoreDriver, StoreKey, StoreLike, + TrueValue, UploadSizeInfo, }; use pretty_assertions::assert_eq; use redis::{PushInfo, RedisError, Value, make_extension_error}; use redis_test::{MockCmd, MockRedisConnection}; +use tokio::sync::mpsc::UnboundedReceiver; use tokio::time::{sleep, timeout}; use tracing::{Instrument, info, info_span}; @@ -84,9 +87,10 @@ async fn fake_redis_sentinel_master_stream_with_script() -> u16 { .await } -async fn make_mock_store_with_prefix( +async fn make_mock_store_with_prefix_and_subscriber_channel( mut commands: Vec, key_prefix: String, + subscriber_channel: UnboundedReceiver, ) -> RedisStore> { commands.insert( 0, @@ -97,7 +101,6 @@ async fn make_mock_store_with_prefix( ); let mock_connection = MockRedisConnection::new(commands); let manager = ClusterRedisManager::new(mock_connection).await.unwrap(); - let (_tx, rx) = tokio::sync::mpsc::unbounded_channel(); RedisStore::new_from_builder_and_parts( None, mock_uuid_generator, @@ -108,13 +111,21 @@ async fn make_mock_store_with_prefix( DEFAULT_MAX_PERMITS, DEFAULT_MAX_COUNT_PER_CURSOR, Duration::from_secs(4), - rx, + subscriber_channel, manager, ) .await .unwrap() } +async fn make_mock_store_with_prefix( + commands: Vec, + key_prefix: String, +) -> RedisStore> { + let (_tx, rx) = tokio::sync::mpsc::unbounded_channel(); + make_mock_store_with_prefix_and_subscriber_channel(commands, key_prefix, rx).await +} + #[nativelink_test] async fn upload_and_get_data() -> Result<(), Error> { // Construct the data we want to send. Since it's small, we expect it to be sent in a single chunk. @@ -133,20 +144,18 @@ async fn upload_and_get_data() -> Result<(), Error> { // Append the real value to the temp key. MockCmd::new( redis::cmd("SETRANGE") - .arg(temp_key.clone()) + .arg(&temp_key) .arg(0) .arg(data.to_vec()), Ok(Value::Int(0)), ), MockCmd::new( - redis::cmd("STRLEN").arg(temp_key.clone()), - Ok(Value::Int(data.len() as i64)), + redis::cmd("STRLEN").arg(&temp_key), + Ok(Value::Int(data.len().try_into().unwrap_or(i64::MAX))), ), // Move the data from the fake key to the real key. MockCmd::new( - redis::cmd("RENAME") - .arg(temp_key.clone()) - .arg(real_key.clone()), + redis::cmd("RENAME").arg(&temp_key).arg(&real_key), Ok(Value::Nil), ), // The second set of commands are for retrieving the data from the key. @@ -154,9 +163,9 @@ async fn upload_and_get_data() -> Result<(), Error> { MockCmd::with_values( redis::pipe() .cmd("STRLEN") - .arg(real_key.clone()) + .arg(&real_key) .cmd("EXISTS") - .arg(real_key.clone()), + .arg(&real_key), Ok(vec![Value::Int(2), Value::Boolean(true)]), ), // Retrieve the data from the real key. @@ -215,7 +224,7 @@ async fn update_retries_after_transient_zero_length() -> Result<(), Error> { // After re-resolving the master, the retry sees the real length. MockCmd::new( redis::cmd("STRLEN").arg(temp_key.clone()), - Ok(Value::Int(data.len() as i64)), + Ok(Value::Int(data.len().try_into().unwrap_or(i64::MAX))), ), MockCmd::new( redis::cmd("RENAME").arg(temp_key).arg(real_key), @@ -339,7 +348,7 @@ async fn update_retries_after_transient_write_error() -> Result<(), Error> { ), MockCmd::new( redis::cmd("STRLEN").arg(temp_key.clone()), - Ok(Value::Int(data.len() as i64)), + Ok(Value::Int(data.len().try_into().unwrap_or(i64::MAX))), ), MockCmd::new( redis::cmd("RENAME").arg(temp_key).arg(real_key), @@ -375,7 +384,7 @@ async fn upload_and_get_data_with_prefix() -> Result<(), Error> { ), MockCmd::new( redis::cmd("STRLEN").arg(temp_key.clone()), - Ok(Value::Int(data.len() as i64)), + Ok(Value::Int(data.len().try_into().unwrap_or(i64::MAX))), ), MockCmd::new( redis::cmd("RENAME").arg(temp_key).arg(real_key.clone()), @@ -473,7 +482,7 @@ async fn test_large_downloads_are_chunked() -> Result<(), Error> { ), MockCmd::new( redis::cmd("STRLEN").arg(temp_key.clone()), - Ok(Value::Int(data.len() as i64)), + Ok(Value::Int(data.len().try_into().unwrap_or(i64::MAX))), ), MockCmd::new( redis::cmd("RENAME").arg(temp_key).arg(real_key.clone()), @@ -496,15 +505,15 @@ async fn test_large_downloads_are_chunked() -> Result<(), Error> { redis::cmd("GETRANGE") .arg(real_key.clone()) .arg(0) - .arg(READ_CHUNK_SIZE as i64 - 1), + .arg(READ_CHUNK_SIZE.try_into().unwrap_or(i64::MAX) - 1), Ok(Value::BulkString(data.slice(..READ_CHUNK_SIZE).into())), ), MockCmd::new( // Similar GETRANGE index shenanigans here. redis::cmd("GETRANGE") .arg(real_key) - .arg(READ_CHUNK_SIZE as i64) - .arg(data.len() as i64 - 1), + .arg(READ_CHUNK_SIZE.try_into().unwrap_or(i64::MAX)) + .arg(data.len().try_into().unwrap_or(i64::MAX) - 1), Ok(Value::BulkString(data.slice(READ_CHUNK_SIZE..).into())), ), ]; @@ -566,7 +575,7 @@ async fn yield_between_sending_packets_in_update() -> Result<(), Error> { ), MockCmd::new( redis::cmd("STRLEN").arg(temp_key.clone()), - Ok(Value::Int(data.len() as i64)), + Ok(Value::Int(data.len().try_into().unwrap_or(i64::MAX))), ), MockCmd::new( redis::cmd("RENAME").arg(temp_key).arg(real_key.clone()), @@ -584,21 +593,29 @@ async fn yield_between_sending_packets_in_update() -> Result<(), Error> { redis::cmd("GETRANGE") .arg(real_key.clone()) .arg(0) - .arg((DEFAULT_READ_CHUNK_SIZE - 1) as i64), + .arg((DEFAULT_READ_CHUNK_SIZE - 1).try_into().unwrap_or(i64::MAX)), Ok(Value::BulkString(data.clone().to_vec())), ), MockCmd::new( redis::cmd("GETRANGE") .arg(real_key.clone()) - .arg(DEFAULT_READ_CHUNK_SIZE as i64) - .arg((DEFAULT_READ_CHUNK_SIZE * 2 - 1) as i64), + .arg(DEFAULT_READ_CHUNK_SIZE.try_into().unwrap_or(i64::MAX)) + .arg( + (DEFAULT_READ_CHUNK_SIZE * 2 - 1) + .try_into() + .unwrap_or(i64::MAX), + ), Ok(Value::BulkString(data.clone().to_vec())), ), MockCmd::new( redis::cmd("GETRANGE") .arg(real_key) - .arg((DEFAULT_READ_CHUNK_SIZE * 2) as i64) - .arg((data_p1.len() + data_p2.len() - 1) as i64), + .arg((DEFAULT_READ_CHUNK_SIZE * 2).try_into().unwrap_or(i64::MAX)) + .arg( + (data_p1.len() + data_p2.len() - 1) + .try_into() + .unwrap_or(i64::MAX), + ), Ok(Value::BulkString(data.clone().to_vec())), ), ]; @@ -653,7 +670,7 @@ async fn zero_len_items_exist_check() -> Result<(), Error> { redis::cmd("GETRANGE") .arg(real_key.clone()) .arg(0) - .arg(DEFAULT_READ_CHUNK_SIZE as i64 - 1), + .arg(DEFAULT_READ_CHUNK_SIZE.try_into().unwrap_or(i64::MAX) - 1), Ok(Value::BulkString(vec![])), ), MockCmd::new(redis::cmd("EXISTS").arg(real_key), Ok(Value::Int(0))), @@ -1863,7 +1880,8 @@ fn test_search_by_index_skips_int_from_cursor_read() -> Result<(), Error> { #[nativelink_test] async fn no_items_from_none_subscription_channel() -> Result<(), Error> { let (_tx, rx) = tokio::sync::mpsc::unbounded_channel(); - let subscription_manager = RedisSubscriptionManager::new(rx); + let subscription_manager = + RedisSubscriptionManager::new(rx, Arc::new(async_lock::Mutex::new(vec![])), String::new()); // To give the stream enough time to get polled sleep(Duration::from_secs(1)).await; @@ -1882,7 +1900,8 @@ async fn no_items_from_none_subscription_channel() -> Result<(), Error> { #[nativelink_test] async fn send_messages_to_subscription_channel() -> Result<(), Error> { let (tx, rx) = tokio::sync::mpsc::unbounded_channel(); - let subscription_manager = RedisSubscriptionManager::new(rx); + let subscription_manager = + RedisSubscriptionManager::new(rx, Arc::new(async_lock::Mutex::new(vec![])), String::new()); tx.send(PushInfo { kind: redis::PushKind::PSubscribe, @@ -1922,6 +1941,10 @@ async fn send_messages_to_subscription_channel() -> Result<(), Error> { // Because otherwise it gets dropped immediately, and we need it to live to do things drop(subscription_manager); + assert!(logs_contain( + "PSubscribe, ignore push_info=PushInfo { kind: PSubscribe, data: [bulk-string('\"scheduler_key_change\"'), int(1)] }" + )); + Ok(()) } @@ -2041,7 +2064,8 @@ impl SchedulerStoreKeyProvider for TestSubKey { #[nativelink_test] async fn redis_subscription_single_drop_is_silent() -> Result<(), Error> { let (_tx, rx) = tokio::sync::mpsc::unbounded_channel(); - let manager = RedisSubscriptionManager::new(rx); + let manager = + RedisSubscriptionManager::new(rx, Arc::new(async_lock::Mutex::new(vec![])), String::new()); let sub = manager.subscribe(TestSubKey("solo-key".to_string()))?; drop(sub); @@ -2060,7 +2084,8 @@ async fn redis_subscription_single_drop_is_silent() -> Result<(), Error> { #[nativelink_test] async fn redis_subscription_drop_one_of_two_keeps_publisher() -> Result<(), Error> { let (_tx, rx) = tokio::sync::mpsc::unbounded_channel(); - let manager = RedisSubscriptionManager::new(rx); + let manager = + RedisSubscriptionManager::new(rx, Arc::new(async_lock::Mutex::new(vec![])), String::new()); let key = "shared-key"; let sub_a = manager.subscribe(TestSubKey(key.to_string()))?; @@ -2088,7 +2113,8 @@ async fn redis_subscription_drop_one_of_two_keeps_publisher() -> Result<(), Erro async fn redis_subscription_concurrent_drops_no_absence_warn() -> Result<(), Error> { const ITERATIONS: usize = 200; let (_tx, rx) = tokio::sync::mpsc::unbounded_channel(); - let manager = RedisSubscriptionManager::new(rx); + let manager = + RedisSubscriptionManager::new(rx, Arc::new(async_lock::Mutex::new(vec![])), String::new()); for i in 0..ITERATIONS { let key = format!("race-key-{i}"); @@ -2122,7 +2148,8 @@ async fn redis_subscription_concurrent_drops_no_absence_warn() -> Result<(), Err #[nativelink_test] async fn redis_subscription_resubscribe_after_drop_creates_fresh_publisher() -> Result<(), Error> { let (_tx, rx) = tokio::sync::mpsc::unbounded_channel(); - let manager = RedisSubscriptionManager::new(rx); + let manager = + RedisSubscriptionManager::new(rx, Arc::new(async_lock::Mutex::new(vec![])), String::new()); let key = "cycle-key"; let sub_a = manager.subscribe(TestSubKey(key.to_string()))?; @@ -2148,3 +2175,289 @@ async fn redis_subscription_resubscribe_after_drop_creates_fresh_publisher() -> drop(manager); Ok(()) } + +#[derive(Debug)] +struct LoggingRemoveCallback {} + +impl RemoveItemCallback for LoggingRemoveCallback { + fn callback<'a>( + &'a self, + store_key: StoreKey<'a>, + ) -> Pin + Send + 'a>> { + info!(?store_key, "Callback for removed item"); + Box::pin(async {}) + } +} + +async fn callback_for_eviction_core( + logs_contain: F, + notify_keyspace_events: &[u8], +) -> Result<(), Error> +where + F: Fn(&str) -> bool, +{ + let redis_span = info_span!("redis"); + + let mut responses = add_lua_version_script(fake_redis_stream()); + add_to_response( + &mut responses, + redis::cmd("CONFIG") + .arg("GET") + .arg("notify-keyspace-events"), + vec![Value::Map(vec![( + Value::BulkString(b"notify-keyspace-events".into()), + Value::BulkString(notify_keyspace_events.into()), + )])], + ); + add_to_response( + &mut responses, + redis::cmd("PSUBSCRIBE").arg("__key*__:*"), + vec![Value::Nil], + ); + let redis_port = make_fake_redis_with_responses(responses) + .instrument(redis_span) + .await; + let spec = RedisSpec { + addresses: vec![format!("redis://127.0.0.1:{redis_port}/")], + mode: RedisMode::Standard, + ..Default::default() + }; + let mut raw_store = + Arc::into_inner(RedisStore::new_standard(spec).await.expect("Working spec")).unwrap(); + raw_store.replace_temp_name_generator(mock_uuid_generator); + let store = Arc::new(raw_store); + store.register_remove_callback(Arc::new(LoggingRemoveCallback {}))?; + + timeout(Duration::from_secs(3), async move { + loop { + if logs_contain("new psubscribe complete pattern=\"__key*__:*\" new_subscription=true") + { + break; + } + sleep(Duration::from_millis(100)).await; + } + }) + .await?; + Ok(()) +} + +#[nativelink_test] +async fn callback_for_eviction_not_enabled() -> Result<(), Error> { + callback_for_eviction_core(logs_contain, b"").await?; + assert!(logs_contain( + "notify-keyspace-events not enabled for Redis, will fail to get remove callbacks" + )); + Ok(()) +} + +#[nativelink_test] +async fn callback_for_eviction_no_keyspace() -> Result<(), Error> { + callback_for_eviction_core(logs_contain, b"E").await?; + assert!(logs_contain( + "notify-keyspace-events does not contain 'K' so won't get keyspace events we need for eviction events notify_keyspace_events=\"E\"" + )); + Ok(()) +} + +#[nativelink_test] +async fn callback_for_eviction_no_all() -> Result<(), Error> { + callback_for_eviction_core(logs_contain, b"K").await?; + assert!(logs_contain( + "notify-keyspace-events does not contain 'A' so we won't get eviction events notify_keyspace_events=\"K\"" + )); + Ok(()) +} + +#[nativelink_test] +async fn callback_for_eviction_good() -> Result<(), Error> { + callback_for_eviction_core(logs_contain, b"KA").await?; + assert!(!logs_contain("ERROR")); + Ok(()) +} + +#[nativelink_test] +async fn send_eviction_to_subscription_channel() -> Result<(), Error> { + let (tx, rx) = tokio::sync::mpsc::unbounded_channel(); + let subscription_manager = RedisSubscriptionManager::new( + rx, + Arc::new(async_lock::Mutex::new(vec![Arc::new( + LoggingRemoveCallback {}, + )])), + String::new(), + ); + + tx.send(PushInfo { + kind: redis::PushKind::PMessage, + data: vec![ + Value::BulkString("demo_pattern".into()), + Value::BulkString("keyprefix:test-eviction".into()), + Value::BulkString("evicted".into()), + ], + }) + .unwrap(); + + timeout(Duration::from_secs(5), async { + loop { + assert!(!logs_contain("ERROR")); + if logs_contain("Callback for removed item store_key=Str(\"test-eviction\")") { + break; + } + sleep(Duration::from_millis(100)).await; + } + }) + .await + .unwrap(); + + // Because otherwise it gets dropped immediately, and we need it to live to do things + drop(subscription_manager); + + assert!(logs_contain( + "Eviction key eviction_key=\"keyprefix:test-eviction\"" + )); + + Ok(()) +} + +async fn store_key_coding_round_trip_core(key_prefix: String) -> Result<(), Error> { + let store = make_mock_store_with_prefix(vec![], key_prefix.clone()).await; + + for key in [ + StoreKey::new_str("foo"), + StoreKey::Digest(DigestInfo::zero_digest()), + StoreKey::Digest(DigestInfo::new([99u8; 32], 512)), + ] { + assert_eq!(key, decode_key(&key_prefix, store.encode_key(&key))?); + } + Ok(()) +} + +#[nativelink_test] +async fn store_key_coding_round_trip() -> Result<(), Error> { + store_key_coding_round_trip_core(String::new()).await +} + +#[nativelink_test] +async fn store_key_coding_round_trip_with_prefix() -> Result<(), Error> { + store_key_coding_round_trip_core(String::from("demo")).await +} + +async fn evict_keys_for_existence_cache_core( + prefix: String, + logs_contain: F, +) -> Result<(), Error> +where + F: Fn(&str) -> bool + Send + Sync, +{ + let spec = ExistenceCacheSpec { + backend: StoreSpec::RedisStore(RedisSpec::default()), // Note: Not used. + eviction_policy: Option::default(), // Not evicting here, we'll evict from Redis + }; + let (tx, rx) = tokio::sync::mpsc::unbounded_channel(); + + let data = Bytes::from_static(b"14"); + let digest = DigestInfo::try_new(VALID_HASH1, 2)?; + let real_key = format!("{prefix}{digest}"); + let temp_key = make_temp_key(&real_key); + + let commands = vec![ + MockCmd::new( + redis::cmd("CONFIG") + .arg("GET") + .arg("notify-keyspace-events"), + Ok(Value::Map(vec![( + Value::BulkString(b"notify-keyspace-events".into()), + Value::BulkString(b"KA".into()), + )])), + ), + MockCmd::with_values( + redis::pipe() + .cmd("STRLEN") + .arg(&real_key) + .cmd("EXISTS") + .arg(&real_key), + Ok(vec![Value::Int(2), Value::Boolean(true)]), + ), + MockCmd::new( + redis::cmd("SETRANGE") + .arg(&temp_key) + .arg(0) + .arg(data.to_vec()), + Ok(Value::Int(0)), + ), + MockCmd::new( + redis::cmd("STRLEN").arg(&temp_key), + Ok(Value::Int(data.len().try_into().unwrap_or(i64::MAX))), + ), + // Move the data from the fake key to the real key. + MockCmd::new( + redis::cmd("RENAME").arg(&temp_key).arg(&real_key), + Ok(Value::Nil), + ), + // Retrieve the data from the real key. + MockCmd::new( + redis::cmd("GETRANGE").arg(&real_key).arg(0).arg(1), + Ok(Value::BulkString(b"14".to_vec())), + ), + ]; + + let redis_store = Arc::new( + make_mock_store_with_prefix_and_subscriber_channel(commands, prefix.clone(), rx).await, + ); + let existence_store = ExistenceCacheStore::new(&spec, Store::new(redis_store.clone())); + + // Wait for subscription first so the mock redis commands are in the right order + timeout(Duration::from_secs(5), async { + loop { + assert!(!logs_contain("ERROR")); + if logs_contain("Attempting to subscribe to eviction events") { + break; + } + sleep(Duration::from_millis(100)).await; + } + }) + .await + .unwrap(); + + existence_store + .update_oneshot(digest, data.clone()) + .await + .unwrap(); + + tx.send(PushInfo { + kind: redis::PushKind::PMessage, + data: vec![ + Value::BulkString("demo_pattern".into()), + Value::BulkString(format!("keyprefix:{real_key}").into()), + Value::BulkString("evicted".into()), + ], + }) + .unwrap(); + + timeout(Duration::from_secs(5), async { + loop { + assert!(!logs_contain("ERROR")); + if logs_contain("Evicting (direct remove) key=DigestInfo(\"3031323334353637383961626364656630303030303030303030303030303030-2\")") { + break; + } + sleep(Duration::from_millis(100)).await; + } + }) + .await + .unwrap(); + + assert!(logs_contain(&format!( + "Eviction key eviction_key=\"keyprefix:{prefix}3031323334353637383961626364656630303030303030303030303030303030-2\"" + ))); + + Ok(()) +} + +#[nativelink_test] +async fn evict_keys_for_existence_cache_no_prefix() -> Result<(), Error> { + evict_keys_for_existence_cache_core(String::new(), logs_contain).await +} + +#[nativelink_test] +async fn evict_keys_for_existence_cache_prefix() -> Result<(), Error> { + evict_keys_for_existence_cache_core(String::from("demo_prefix:"), logs_contain).await +} diff --git a/nativelink-store/tests/verify_store_test.rs b/nativelink-store/tests/verify_store_test.rs index dca6b4802..97cc07c81 100644 --- a/nativelink-store/tests/verify_store_test.rs +++ b/nativelink-store/tests/verify_store_test.rs @@ -179,7 +179,11 @@ async fn verify_sha256_hash_true_succeeds_on_update() -> Result<(), Error> { ); let digest = DigestInfo::try_new(HASH, 3).unwrap(); - let result = store.update_oneshot(digest, VALUE.into()).await; + let result = store + .update_oneshot(digest, VALUE.into()) + .instrument(info_span!("update_oneshot")) + .with_context(make_ctx_for_hash_func(DigestHasherFunc::Sha256)?) + .await; assert_eq!(result, Ok(()), "Expected success, got: {:?}", result); assert_eq!( inner_store.has(digest).await, @@ -207,10 +211,15 @@ async fn verify_sha256_hash_true_fails_on_update() -> Result<(), Error> { ); let digest = DigestInfo::try_new(HASH, 3).unwrap(); - let result = store.update_oneshot(digest, VALUE.into()).await; + let result = store + .update_oneshot(digest, VALUE.into()) + .instrument(info_span!("update_oneshot")) + .with_context(make_ctx_for_hash_func(DigestHasherFunc::Sha256)?) + .await; let err = result.unwrap_err().to_string(); - let expected_err = - format!("Hashes do not match, got: {HASH} but digest hash was {ACTUAL_HASH}"); + let expected_err = format!( + "Hash verification using SHA256 failed: client declared SHA256:{HASH}, but the server computed SHA256:{ACTUAL_HASH}" + ); assert!( err.contains(&expected_err), "Error should contain '{expected_err}', got: {err:?}" @@ -283,8 +292,9 @@ async fn verify_blake3_hash_true_fails_on_update() -> Result<(), Error> { // let result = store.update_oneshot(digest, VALUE.into()).await; let err = result.unwrap_err().to_string(); - let expected_err = - format!("Hashes do not match, got: {HASH} but digest hash was {ACTUAL_HASH}"); + let expected_err = format!( + "Hash verification using BLAKE3 failed: client declared BLAKE3:{HASH}, but the server computed BLAKE3:{ACTUAL_HASH}" + ); assert!( err.contains(&expected_err), "Error should contain '{expected_err}', got: {err:?}" @@ -360,7 +370,11 @@ async fn verify_size_and_hash_succeeds_on_small_data() -> Result<(), Error> { ); let digest = DigestInfo::try_new(HASH, 3).unwrap(); - let result = store.update_oneshot(digest, VALUE.into()).await; + let result = store + .update_oneshot(digest, VALUE.into()) + .instrument(info_span!("update_oneshot")) + .with_context(make_ctx_for_hash_func(DigestHasherFunc::Sha256)?) + .await; assert_eq!(result, Ok(()), "Expected success, got: {:?}", result); assert_eq!( inner_store.has(digest).await, diff --git a/nativelink-store/tests/zstd_store_test.rs b/nativelink-store/tests/zstd_store_test.rs new file mode 100644 index 000000000..61665be62 --- /dev/null +++ b/nativelink-store/tests/zstd_store_test.rs @@ -0,0 +1,2303 @@ +// Copyright 2026 The NativeLink Authors. All rights reserved. +// +// Licensed under the Functional Source License, Version 1.1, Apache 2.0 Future License (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// See LICENSE file for details +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use core::pin::Pin; +use core::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; +use core::time::Duration; +use std::ffi::OsString; +use std::sync::Arc; + +use async_trait::async_trait; +use bytes::Bytes; +use nativelink_config::stores::{ + CacheMetricsSpec, CompressionAlgorithm, CompressionSpec, DedupSpec, ExistenceCacheSpec, + FastSlowSpec, FilesystemSpec, MemorySpec, NoopSpec, RefSpec, ShardConfig, ShardSpec, + SizePartitioningSpec, StoreDirection, StoreSpec, ZstdConfig, +}; +use nativelink_error::{Code, Error, make_err}; +use nativelink_macro::nativelink_test; +use nativelink_metric::MetricsComponent; +use nativelink_store::cache_metrics_store::CacheMetricsStore; +use nativelink_store::cas_utils::{ZERO_BYTE_DIGESTS, is_zero_digest}; +use nativelink_store::dedup_store::DedupStore; +use nativelink_store::default_store_factory::store_factory; +use nativelink_store::existence_cache_store::ExistenceCacheStore; +use nativelink_store::fast_slow_store::FastSlowStore; +use nativelink_store::filesystem_store::{FileEntryImpl, FilesystemStore}; +use nativelink_store::memory_store::MemoryStore; +use nativelink_store::ref_store::RefStore; +use nativelink_store::shard_store::ShardStore; +use nativelink_store::size_partitioning_store::SizePartitioningStore; +use nativelink_store::store_manager::StoreManager; +use nativelink_store::zstd_store::ZstdStore; +use nativelink_util::buf_channel::{ + DropCloserReadHalf, DropCloserWriteHalf, make_buf_channel_pair, +}; +use nativelink_util::common::{DigestInfo, make_temp_path}; +use nativelink_util::digest_hasher::{DigestHasher, DigestHasherFunc, make_ctx_for_hash_func}; +use nativelink_util::fs::FileSlot; +use nativelink_util::health_utils::{HealthStatusIndicator, default_health_status_indicator}; +use nativelink_util::store_trait::{ + RemoveCallback, Store, StoreDriver, StoreKey, StoreLike, UploadSizeInfo, WireCompressor, +}; +use nativelink_util::{background_spawn, spawn}; +use opentelemetry::context::FutureExt; +use pretty_assertions::assert_eq; +use sha2::{Digest, Sha256}; +use tokio::io::{AsyncReadExt, AsyncSeekExt}; + +#[derive(Default, MetricsComponent)] +struct RecordingStore { + update_count: AtomicUsize, + /// Total number of keys the inner store observed via `has_with_results`. + has_key_count: AtomicUsize, + /// Set if any zero-byte digest was ever forwarded to the inner store. + has_saw_zero: AtomicBool, +} + +#[async_trait] +impl StoreDriver for RecordingStore { + async fn post_init(self: Arc) -> Result<(), Error> { + Ok(()) + } + + async fn has_with_results( + self: Pin<&Self>, + keys: &[StoreKey<'_>], + _results: &mut [Option], + ) -> Result<(), Error> { + self.has_key_count.fetch_add(keys.len(), Ordering::Relaxed); + if keys.iter().any(|key| is_zero_digest(key.borrow())) { + self.has_saw_zero.store(true, Ordering::Relaxed); + } + Ok(()) + } + + async fn update( + self: Pin<&Self>, + _key: StoreKey<'_>, + _reader: DropCloserReadHalf, + _size_info: UploadSizeInfo, + ) -> Result { + self.update_count.fetch_add(1, Ordering::Relaxed); + Ok(0) + } + + async fn get_part( + self: Pin<&Self>, + _key: StoreKey<'_>, + _writer: &mut DropCloserWriteHalf, + _offset: u64, + _length: Option, + ) -> Result<(), Error> { + Err(make_err!(Code::NotFound, "Not found")) + } + + fn inner_store(&self, _key: Option) -> &dyn StoreDriver { + self + } + + fn as_any(&self) -> &(dyn core::any::Any + Sync + Send + 'static) { + self + } + + fn as_any_arc(self: Arc) -> Arc { + self + } + + fn register_remove_callback(self: Arc, _callback: RemoveCallback) -> Result<(), Error> { + Ok(()) + } +} + +default_health_status_indicator!(RecordingStore); + +/// A store wrapper that stores data in an inner [`MemoryStore`] but, on +/// `get_part`, replays the physical bytes in many small `send` calls. This forces +/// the `ZstdStore` physical channel (a 2-slot buffer) to back up so the drain +/// behavior of the decode path is actually exercised. A real streaming backend +/// (S3, filesystem) chunks its output the same way. +#[derive(MetricsComponent)] +struct ChunkingStore { + #[metric(group = "inner")] + inner: Store, + chunk_size: usize, +} + +#[async_trait] +impl StoreDriver for ChunkingStore { + async fn post_init(self: Arc) -> Result<(), Error> { + Ok(()) + } + + async fn has_with_results( + self: Pin<&Self>, + keys: &[StoreKey<'_>], + results: &mut [Option], + ) -> Result<(), Error> { + self.inner.has_with_results(keys, results).await + } + + async fn update( + self: Pin<&Self>, + key: StoreKey<'_>, + reader: DropCloserReadHalf, + size_info: UploadSizeInfo, + ) -> Result { + self.inner + .as_store_driver_pin() + .update(key, reader, size_info) + .await + } + + async fn get_part( + self: Pin<&Self>, + key: StoreKey<'_>, + writer: &mut DropCloserWriteHalf, + offset: u64, + length: Option, + ) -> Result<(), Error> { + // Ignore offset/length: ZstdStore always requests the full physical blob. + let _ = (offset, length); + let full = self.inner.get_part_unchunked(key.borrow(), 0, None).await?; + for chunk in full.chunks(self.chunk_size) { + writer.send(Bytes::copy_from_slice(chunk)).await?; + } + writer.send_eof()?; + Ok(()) + } + + fn inner_store(&self, _key: Option) -> &dyn StoreDriver { + self + } + + fn as_any(&self) -> &(dyn core::any::Any + Sync + Send + 'static) { + self + } + + fn as_any_arc(self: Arc) -> Arc { + self + } + + fn register_remove_callback(self: Arc, _callback: RemoveCallback) -> Result<(), Error> { + Ok(()) + } +} + +default_health_status_indicator!(ChunkingStore); + +const TEMP_PATH: &str = "/tmp/nativelink-zstd-store-test"; + +fn spec() -> ZstdConfig { + ZstdConfig { + temp_path: TEMP_PATH.to_string(), + max_compressed_upload_size: 512 * 1024 * 1024, + // Exercise the staging path; the inline path has dedicated tests. + max_inline_commit_size: 1, + ..ZstdConfig::default() + } +} + +fn digest_for(data: &[u8]) -> DigestInfo { + let hash: [u8; 32] = Sha256::digest(data).into(); + DigestInfo::new(hash, data.len() as u64) +} + +#[nativelink_test] +async fn identity_round_trip() -> Result<(), Error> { + const DATA: &[u8] = b"hello zstd store, this compresses a bit aaaaaaaaaaaaaaaaaaaa"; + + let inner = Store::new(MemoryStore::new(&MemorySpec::default())); + let store = Store::new(ZstdStore::new(&spec(), inner)?); + + let digest = digest_for(DATA); + + store.update_oneshot(digest, DATA.into()).await?; + let got = store.get_part_unchunked(digest, 0, None).await?; + assert_eq!(&got[..], DATA, "Expected round-tripped data to match"); + Ok(()) +} + +#[nativelink_test] +async fn identity_round_trip_partial_read() -> Result<(), Error> { + const DATA: &[u8] = b"0123456789abcdefghijklmnopqrstuvwxyz0123456789"; + + let inner = Store::new(MemoryStore::new(&MemorySpec::default())); + let store = Store::new(ZstdStore::new(&spec(), inner)?); + + let digest = digest_for(DATA); + + store.update_oneshot(digest, DATA.into()).await?; + let got = store.get_part_unchunked(digest, 5, Some(10)).await?; + assert_eq!(&got[..], &DATA[5..15], "Expected partial read to match"); + Ok(()) +} + +#[nativelink_test] +async fn zero_digest_read_returns_empty() -> Result<(), Error> { + let inner = Store::new(MemoryStore::new(&MemorySpec::default())); + let store = Store::new(ZstdStore::new(&spec(), inner)?); + + for digest in ZERO_BYTE_DIGESTS { + let got = store.get_part_unchunked(digest, 0, None).await?; + assert_eq!(got.len(), 0, "Expected zero-digest read to be empty"); + } + Ok(()) +} + +#[nativelink_test] +async fn zero_digest_write_empty_skips_inner() -> Result<(), Error> { + let recording = Arc::new(RecordingStore::default()); + let store = Store::new(ZstdStore::new(&spec(), Store::new(recording.clone()))?); + + for digest in ZERO_BYTE_DIGESTS { + store.update_oneshot(digest, Bytes::new()).await?; + } + // The inner store's update must never be called for zero digests. + assert_eq!( + recording.update_count.load(Ordering::Relaxed), + 0, + "Zero digest must not reach the inner store" + ); + Ok(()) +} + +#[nativelink_test] +async fn str_key_is_rejected() -> Result<(), Error> { + let inner = Store::new(MemoryStore::new(&MemorySpec::default())); + let store = Store::new(ZstdStore::new(&spec(), inner)?); + + let keys = [StoreKey::new_str("some-string-key")]; + let mut results = [None]; + let err = store + .has_with_results(&keys, &mut results) + .await + .expect_err("Expected a Str key to be rejected"); + assert!( + err.to_string().contains("only supports digest keys"), + "Unexpected error: {err}" + ); + Ok(()) +} + +#[nativelink_test] +async fn has_reports_uncompressed_digest_size() -> Result<(), Error> { + let inner = Store::new(MemoryStore::new(&MemorySpec::default())); + let store = Store::new(ZstdStore::new(&spec(), inner)?); + + // Highly compressible payload so the physical zstd size differs from the + // uncompressed digest size. + let data = vec![0u8; 4096]; + let digest = digest_for(&data); + + store.update_oneshot(digest, data.clone().into()).await?; + let reported = store.has(digest).await?; + assert_eq!( + reported, + Some(data.len() as u64), + "has must report the uncompressed digest size, not the physical zstd size" + ); + Ok(()) +} + +#[nativelink_test] +async fn ranged_read_large_blob_drains_inner_stream() -> Result<(), Error> { + // Regression for the partial-read drain bug: a ranged read on a blob whose + // physical (compressed) stream is larger than the 2-slot channel buffer must + // still return the requested bytes. Before the fix the decode loop returned + // as soon as the requested `length` was produced, dropping the physical + // reader while the spawned inner `get` was still streaming compressed bytes; + // its next `send` then failed with "receiver disconnected" and the whole + // get_part surfaced that error instead of the requested bytes. + // + // Use a repeating-but-nontrivial pattern (not all-zero) so the compressed + // stream is large enough to span multiple channel sends. + let mut data = Vec::with_capacity(1024 * 1024); + let mut counter: u32 = 0x1234_5678; + while data.len() < 1024 * 1024 { + counter = counter.wrapping_mul(1_664_525).wrapping_add(1_013_904_223); + data.extend_from_slice(&counter.to_le_bytes()); + } + + // A chunking inner store forces the physical stream to span many channel + // sends (16 KiB each) so the 2-slot channel backs up while the decoder is + // still consuming; this is what makes an early return drop the reader and + // fail the inner get. + let inner = Store::new(Arc::new(ChunkingStore { + inner: Store::new(MemoryStore::new(&MemorySpec::default())), + chunk_size: 16 * 1024, + })); + let store = Store::new(ZstdStore::new(&spec(), inner)?); + + let digest = digest_for(&data); + store.update_oneshot(digest, data.clone().into()).await?; + + let offset = 300 * 1024; + let length = 100 * 1024; + let got = store + .get_part_unchunked(digest, offset as u64, Some(length as u64)) + .await?; + assert_eq!( + &got[..], + &data[offset..offset + length], + "Ranged read of a large blob must return the requested sub-range" + ); + Ok(()) +} + +#[nativelink_test] +async fn corrupt_upload_hash_mismatch_is_rejected_and_not_committed() -> Result<(), Error> { + const DATA: &[u8] = b"the quick brown fox jumps over the lazy dog"; + + let inner = Store::new(MemoryStore::new(&MemorySpec::default())); + let store = Store::new(ZstdStore::new(&spec(), inner.clone())?); + + // Claim a digest whose hash does NOT match the data (but the size matches). + let mut wrong = DATA.to_vec(); + wrong[0] ^= 0xFF; + let bad_digest = digest_for(&wrong); + assert_eq!(bad_digest.size_bytes(), DATA.len() as u64); + + let err = store + .update_oneshot(bad_digest, DATA.into()) + .await + .expect_err("Expected a hash mismatch to be rejected"); + assert_eq!( + err.code, + Code::InvalidArgument, + "Hash mismatch must be InvalidArgument, got: {err}" + ); + + // The blob must NOT have committed to the inner store. + assert_eq!( + store.has(bad_digest).await?, + None, + "A rejected upload must not be visible via the zstd store" + ); + assert_eq!( + inner.has(bad_digest).await?, + None, + "A rejected upload must not commit to the inner store" + ); + Ok(()) +} + +#[nativelink_test] +async fn corrupt_upload_size_mismatch_is_rejected_and_not_committed() -> Result<(), Error> { + const DATA: &[u8] = b"the quick brown fox jumps over the lazy dog"; + + let inner = Store::new(MemoryStore::new(&MemorySpec::default())); + let store = Store::new(ZstdStore::new(&spec(), inner.clone())?); + + // Correct hash, but claim a size larger than the actual data length. + let hash: [u8; 32] = Sha256::digest(DATA).into(); + let bad_digest = DigestInfo::new(hash, DATA.len() as u64 + 10); + + let err = store + .update_oneshot(bad_digest, DATA.into()) + .await + .expect_err("Expected a size mismatch to be rejected"); + assert_eq!( + err.code, + Code::InvalidArgument, + "Size mismatch must be InvalidArgument, got: {err}" + ); + + assert_eq!( + store.has(bad_digest).await?, + None, + "A rejected upload must not be visible via the zstd store" + ); + assert_eq!( + inner.has(bad_digest).await?, + None, + "A rejected upload must not commit to the inner store" + ); + Ok(()) +} + +#[nativelink_test] +async fn new_rejects_out_of_range_compression_levels() -> Result<(), Error> { + let inner = Store::new(MemoryStore::new(&MemorySpec::default())); + + for level in [0, 20] { + let mut bad_spec = spec(); + bad_spec.compression_level = Some(level); + assert!( + ZstdStore::new(&bad_spec, inner.clone()).is_err(), + "compression_level {level} must be rejected" + ); + } + + for level in [Some(1), Some(19), None] { + let mut ok_spec = spec(); + ok_spec.compression_level = level; + assert!( + ZstdStore::new(&ok_spec, inner.clone()).is_ok(), + "compression_level {level:?} must be accepted" + ); + } + Ok(()) +} + +#[nativelink_test] +async fn identity_round_trip_blake3_context() -> Result<(), Error> { + const DATA: &[u8] = b"blake3 round trip payload aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"; + + let inner = Store::new(MemoryStore::new(&MemorySpec::default())); + let store = Store::new(ZstdStore::new(&spec(), inner)?); + + // Compute the BLAKE3 digest of the payload using the context hasher. + let mut hasher = DigestHasherFunc::Blake3.hasher(); + hasher.update(DATA); + let digest = hasher.finalize_digest(); + + // Run both the update and get under a BLAKE3 request-digest context, proving + // the context-hasher mechanism (not the SHA-256 default) is honored. + let ctx = make_ctx_for_hash_func(DigestHasherFunc::Blake3)?; + store + .update_oneshot(digest, DATA.into()) + .with_context(ctx.clone()) + .await?; + let got = store + .get_part_unchunked(digest, 0, None) + .with_context(ctx) + .await?; + assert_eq!(&got[..], DATA, "Expected BLAKE3 round-trip to match"); + Ok(()) +} + +#[nativelink_test] +async fn has_zero_digest_never_touches_inner_store() -> Result<(), Error> { + let recording = Arc::new(RecordingStore::default()); + let store = Store::new(ZstdStore::new(&spec(), Store::new(recording.clone()))?); + + // Mixed batch: two zero digests and one non-zero (absent) digest. + let nonzero = digest_for(b"not present"); + let keys = [ + StoreKey::Digest(ZERO_BYTE_DIGESTS[0]), + StoreKey::Digest(nonzero), + StoreKey::Digest(ZERO_BYTE_DIGESTS[1]), + ]; + let mut results = [None, None, None]; + store.has_with_results(&keys, &mut results).await?; + + assert_eq!( + results, + [Some(0), None, Some(0)], + "Zero digests must report Some(0); absent non-zero must be None" + ); + assert!( + !recording.has_saw_zero.load(Ordering::Relaxed), + "Zero digests must never be forwarded to the inner store" + ); + assert_eq!( + recording.has_key_count.load(Ordering::Relaxed), + 1, + "Only the single non-zero key may reach the inner store" + ); + Ok(()) +} + +#[nativelink_test] +async fn has_mixed_batch_reports_correct_sizes() -> Result<(), Error> { + let inner = Store::new(MemoryStore::new(&MemorySpec::default())); + let store = Store::new(ZstdStore::new(&spec(), inner)?); + + let present = vec![7u8; 4096]; + let present_digest = digest_for(&present); + store + .update_oneshot(present_digest, present.clone().into()) + .await?; + let absent_digest = digest_for(b"definitely absent"); + + let keys = [ + StoreKey::Digest(ZERO_BYTE_DIGESTS[0]), + StoreKey::Digest(present_digest), + StoreKey::Digest(absent_digest), + ]; + let mut results = [None, None, None]; + store.has_with_results(&keys, &mut results).await?; + assert_eq!( + results, + [Some(0), Some(present.len() as u64), None], + "Mixed batch must report zero=Some(0), present=uncompressed size, absent=None" + ); + Ok(()) +} + +#[nativelink_test] +async fn factory_selects_zstd_wire_capability() -> Result<(), Error> { + let store_spec = StoreSpec::Compression(Box::new(CompressionSpec { + backend: StoreSpec::Memory(MemorySpec::default()), + compression_algorithm: CompressionAlgorithm::Zstd(spec()), + })); + let store_manager = Arc::new(StoreManager::new()); + let store = store_factory(&store_spec, &store_manager, None).await?; + assert!( + store.wire_compression_store().is_some(), + "Expected CompressionAlgorithm::Zstd to expose the wire-compression capability" + ); + Ok(()) +} + +// --------------------------------------------------------------------------- +// zstd fast-path (get_zstd / update_zstd / update_zstd_oneshot / get_for_batch) +// --------------------------------------------------------------------------- + +/// A spec pointing at a specific temp dir (for tests that assert temp-dir +/// emptiness and therefore must not share the global staging directory). +/// +/// `max_inline_commit_size: 1` forces even tiny oneshot payloads through the +/// staging path, which is what most tests below are actually asserting about. +/// The inline path has its own tests; see [`spec_for_inline`]. +fn spec_for(temp_path: String) -> ZstdConfig { + ZstdConfig { + temp_path, + max_compressed_upload_size: 512 * 1024 * 1024, + max_inline_commit_size: 1, + ..ZstdConfig::default() + } +} + +/// Like [`spec_for`], but leaves `max_inline_commit_size` at its default so +/// small compressed oneshot payloads take the in-memory commit path. +fn spec_for_inline(temp_path: String) -> ZstdConfig { + ZstdConfig { + temp_path, + max_compressed_upload_size: 512 * 1024 * 1024, + ..ZstdConfig::default() + } +} + +/// Builds a `ZstdStore` over a fresh `MemoryStore`, ensuring the staging dir +/// exists. Returns the concrete store (for the fast-path methods), a `Store` +/// wrapper (for the identity round-trip view), and the inner store (for +/// physical inspection). +async fn build(spec: &ZstdConfig) -> Result<(Arc, Store, Store), Error> { + std::fs::create_dir_all(&spec.temp_path) + .map_err(|e| make_err!(Code::Internal, "Failed to create test temp dir: {e}"))?; + let inner = Store::new(MemoryStore::new(&MemorySpec::default())); + let zstd = ZstdStore::new(spec, inner.clone())?; + let store = Store::new(zstd.clone()); + Ok((zstd, store, inner)) +} + +/// Drive `get_zstd` and collect the full emitted (physical zstd) stream. +async fn collect_zstd(store: &ZstdStore, digest: DigestInfo) -> Result { + let (tx, mut rx) = make_buf_channel_pair(); + let (get_res, collected) = tokio::join!(store.get_zstd(digest, tx), rx.consume(None)); + get_res?; + collected +} + +/// A `DropCloserReadHalf` that yields the given chunks then EOF, fed by a task. +fn reader_from(chunks: Vec) -> DropCloserReadHalf { + let (mut tx, rx) = make_buf_channel_pair(); + background_spawn!("zstd_test_reader_feed", async move { + for chunk in chunks { + if tx.send(chunk).await.is_err() { + return; + } + } + drop(tx.send_eof()); + }); + rx +} + +/// Highly compressible data (a repeated pseudo-random block) whose compressed +/// size is sensitive to the zstd level. +fn compressible_data(len: usize) -> Vec { + let mut block = [0u8; 251]; + let mut state: u32 = 0x9E37_79B1; + for b in &mut block { + state = state.wrapping_mul(1_664_525).wrapping_add(1_013_904_223); + *b = (state >> 24) as u8; + } + let mut out = Vec::with_capacity(len); + while out.len() < len { + let take = (len - out.len()).min(block.len()); + out.extend_from_slice(&block[..take]); + } + out +} + +fn random_bytes(len: usize) -> Vec { + use rand::RngCore; + let mut out = vec![0u8; len]; + rand::rng().fill_bytes(&mut out); + out +} + +/// Data whose compressed size is strongly level-sensitive: two copies of a +/// large incompressible block. Only a large-window (high level) encoder can +/// dedup the second copy, so a low level produces a much larger stream. +fn level_sensitive_data() -> Vec { + let block = random_bytes(512 * 1024); + let mut out = Vec::with_capacity(block.len() * 2); + out.extend_from_slice(&block); + out.extend_from_slice(&block); + out +} + +fn dir_entry_count(path: &str) -> usize { + std::fs::read_dir(path).map_or(0, Iterator::count) +} + +#[nativelink_test] +async fn get_zstd_is_byte_for_byte_passthrough() -> Result<(), Error> { + const DATA: &[u8] = b"passthrough payload aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"; + + let (zstd, _store, _inner) = build(&spec()).await?; + let compressed = zstd::bulk::compress(DATA, 3).unwrap(); + let digest = digest_for(DATA); + + let wire = zstd + .update_zstd_oneshot( + digest, + DigestHasherFunc::Sha256, + Bytes::from(compressed.clone()), + ) + .await?; + assert_eq!(wire, compressed.len() as u64, "wire bytes must match input"); + + let got = collect_zstd(&zstd, digest).await?; + assert_eq!( + &got[..], + &compressed[..], + "get_zstd must return the stored zstd stream byte-for-byte" + ); + let decoded = zstd::stream::decode_all(&got[..]).unwrap(); + assert_eq!( + &decoded[..], + DATA, + "the passthrough stream must decode back" + ); + Ok(()) +} + +#[nativelink_test] +async fn get_zstd_preserves_concatenated_frames() -> Result<(), Error> { + // Two independently-compressed zstd frames concatenated into one stream. + const PART_A: &[u8] = b"first frame content aaaaaaaaaaaaaaaaaaaaaaaa"; + const PART_B: &[u8] = b"second frame content bbbbbbbbbbbbbbbbbbbbbbbb"; + + let frame_a = zstd::bulk::compress(PART_A, 3).unwrap(); + let frame_b = zstd::bulk::compress(PART_B, 3).unwrap(); + let mut concatenated = frame_a.clone(); + concatenated.extend_from_slice(&frame_b); + + let mut raw = PART_A.to_vec(); + raw.extend_from_slice(PART_B); + let digest = digest_for(&raw); + + let (zstd, _store, _inner) = build(&spec()).await?; + // Upload the two frames as separate stream chunks to exercise the streaming + // path; passthrough must not re-frame them. + let wire = zstd + .update_zstd( + digest, + DigestHasherFunc::Sha256, + reader_from(vec![Bytes::from(frame_a), Bytes::from(frame_b)]), + ) + .await?; + assert_eq!(wire, concatenated.len() as u64); + + let got = collect_zstd(&zstd, digest).await?; + assert_eq!( + &got[..], + &concatenated[..], + "passthrough must preserve the exact concatenated-frame bytes" + ); + assert_eq!( + zstd::stream::decode_all(&got[..]).unwrap(), + raw, + "concatenated frames must decode to the concatenated raw content" + ); + Ok(()) +} + +#[nativelink_test] +async fn update_zstd_round_trips_through_identity() -> Result<(), Error> { + const DATA: &[u8] = b"round trip payload cccccccccccccccccccccccccccccccc"; + + let (zstd, store, _inner) = build(&spec()).await?; + let compressed = zstd::bulk::compress(DATA, 3).unwrap(); + let digest = digest_for(DATA); + + let wire = zstd + .update_zstd_oneshot( + digest, + DigestHasherFunc::Sha256, + Bytes::from(compressed.clone()), + ) + .await?; + assert_eq!(wire, compressed.len() as u64); + + let got = store.get_part_unchunked(digest, 0, None).await?; + assert_eq!( + &got[..], + DATA, + "identity read must return the original raw bytes" + ); + Ok(()) +} + +#[nativelink_test] +async fn update_zstd_round_trips_with_blake3() -> Result<(), Error> { + const DATA: &[u8] = b"blake3 compressed upload dddddddddddddddddddddddddddd"; + + let (zstd, store, _inner) = build(&spec()).await?; + let mut hasher = DigestHasherFunc::Blake3.hasher(); + hasher.update(DATA); + let digest = hasher.finalize_digest(); + let compressed = zstd::bulk::compress(DATA, 3).unwrap(); + + zstd.update_zstd_oneshot(digest, DigestHasherFunc::Blake3, Bytes::from(compressed)) + .await?; + let got = store.get_part_unchunked(digest, 0, None).await?; + assert_eq!(&got[..], DATA, "BLAKE3 compressed round-trip must match"); + Ok(()) +} + +#[nativelink_test] +async fn update_zstd_rejects_hash_mismatch() -> Result<(), Error> { + const DATA: &[u8] = b"the quick brown fox jumps over the lazy dog"; + + let temp = make_temp_path("zstd-hash-mismatch"); + let (zstd, _store, inner) = build(&spec_for(temp.clone())).await?; + + // Compress DATA but claim the digest of a different blob of the same size. + let compressed = zstd::bulk::compress(DATA, 3).unwrap(); + let mut other = DATA.to_vec(); + other[0] ^= 0xFF; + let bad_digest = digest_for(&other); + assert_eq!(bad_digest.size_bytes(), DATA.len() as u64); + + let err = zstd + .update_zstd_oneshot( + bad_digest, + DigestHasherFunc::Sha256, + Bytes::from(compressed), + ) + .await + .expect_err("hash mismatch must be rejected"); + assert_eq!( + err.code, + Code::InvalidArgument, + "hash mismatch must be InvalidArgument, got: {err}" + ); + assert_eq!( + inner.has(bad_digest).await?, + None, + "a rejected compressed upload must not commit to the inner store" + ); + assert_eq!( + dir_entry_count(&temp), + 0, + "a rejected upload must not leave staged temp files" + ); + Ok(()) +} + +/// A complete frame followed by the beginning of a second frame is not a +/// complete concatenated zstd stream. `flush` alone accepted this shape, so it +/// is important that both the staged and zero-digest validation paths finalize +/// the decoder at EOF. +#[nativelink_test] +async fn update_zstd_rejects_incomplete_trailing_frame() -> Result<(), Error> { + const DATA: &[u8] = b"complete first frame followed by a truncated second frame"; + + let temp = make_temp_path("zstd-incomplete-trailing-frame"); + let (zstd, _store, inner) = build(&spec_for(temp.clone())).await?; + let digest = digest_for(DATA); + let mut trailing = zstd::bulk::compress(DATA, 3).unwrap(); + let next_frame = zstd::bulk::compress(b"second frame", 3).unwrap(); + trailing.extend_from_slice(&next_frame[..4]); // zstd magic, but no full frame. + + let err = zstd + .update_zstd_oneshot(digest, DigestHasherFunc::Sha256, Bytes::from(trailing)) + .await + .expect_err("a partial trailing zstd frame must be rejected"); + assert_eq!(err.code, Code::InvalidArgument, "got: {err}"); + assert_eq!( + inner.has(digest).await?, + None, + "a stream with an incomplete trailing frame must not commit" + ); + assert_eq!( + dir_entry_count(&temp), + 0, + "a rejected trailing frame must not leave a staging file" + ); + + let zero = ZERO_BYTE_DIGESTS[0]; + let mut empty_then_partial = zstd::bulk::compress(&[], 3).unwrap(); + empty_then_partial.extend_from_slice(&next_frame[..4]); + let err = zstd + .update_zstd_oneshot( + zero, + DigestHasherFunc::Sha256, + Bytes::from(empty_then_partial), + ) + .await + .expect_err("zero-digest validation must reject a partial trailing frame too"); + assert_eq!(err.code, Code::InvalidArgument, "got: {err}"); + Ok(()) +} + +#[nativelink_test] +async fn update_zstd_rejects_oversize_and_cleans_up() -> Result<(), Error> { + let temp = make_temp_path("zstd-oversize"); + let mut spec = spec_for(temp.clone()); + spec.max_compressed_upload_size = 32; // Very small cap. + let (zstd, _store, inner) = build(&spec).await?; + + // Incompressible data so the compressed stream comfortably exceeds 32 bytes. + let data = random_bytes(4096); + let compressed = zstd::bulk::compress(&data, 3).unwrap(); + assert!(compressed.len() as u64 > spec.max_compressed_upload_size); + let digest = digest_for(&data); + + let err = zstd + .update_zstd_oneshot(digest, DigestHasherFunc::Sha256, Bytes::from(compressed)) + .await + .expect_err("oversize upload must be rejected"); + assert_eq!( + err.code, + Code::ResourceExhausted, + "oversize upload must be ResourceExhausted, got: {err}" + ); + assert_eq!( + inner.has(digest).await?, + None, + "an oversize upload must not commit to the inner store" + ); + assert_eq!( + dir_entry_count(&temp), + 0, + "an oversize upload must not leave staged temp files" + ); + Ok(()) +} + +/// Cancelling the async request after the blocking stage creates its file must +/// not leak it. The guard is moved into the detached blocking task before that +/// task can create the path; after EOF lets it finish, dropping its unobserved +/// result closes the descriptor before removing the file. +#[nativelink_test] +async fn cancelled_staging_task_eventually_cleans_its_temp_file() -> Result<(), Error> { + const DATA: &[u8] = b"cancelled staging cleanup payload"; + + let temp = make_temp_path("zstd-cancelled-stage"); + let (zstd, _store, _inner) = build(&spec_for(temp.clone())).await?; + let digest = digest_for(DATA); + let compressed = zstd::bulk::compress(DATA, 3).unwrap(); + let (mut tx, rx) = make_buf_channel_pair(); + let task_store = zstd.clone(); + let task = tokio::spawn(async move { + task_store + .update_zstd(digest, DigestHasherFunc::Sha256, rx) + .await + }); + + tx.send(Bytes::from(compressed)) + .await + .map_err(|e| make_err!(Code::Internal, "failed to feed staging task: {e}"))?; + tokio::time::timeout(Duration::from_secs(1), async { + while dir_entry_count(&temp) == 0 { + tokio::task::yield_now().await; + } + }) + .await + .expect("the blocking stage must create its temp file before EOF"); + + task.abort(); + assert!(task.await.is_err(), "the request task must be cancelled"); + tx.send_eof() + .map_err(|e| make_err!(Code::Internal, "failed to finish staging input: {e}"))?; + + tokio::time::timeout(Duration::from_secs(1), async { + while dir_entry_count(&temp) != 0 { + tokio::task::yield_now().await; + } + }) + .await + .expect("the detached blocking stage must eventually remove its temp file"); + Ok(()) +} + +#[nativelink_test] +async fn update_zstd_recompresses_to_smaller_stream() -> Result<(), Error> { + let data = level_sensitive_data(); + let digest = digest_for(&data); + + // Upload a poorly-compressed (level 1) stream; store re-compresses at 19. + let mut spec = spec_for(make_temp_path("zstd-recompress")); + spec.compression_level = Some(19); + spec.max_recompression_size = 16 * 1024 * 1024; + let (zstd, _store, inner) = build(&spec).await?; + + let poorly = zstd::bulk::compress(&data, 1).unwrap(); + zstd.update_zstd_oneshot( + digest, + DigestHasherFunc::Sha256, + Bytes::from(poorly.clone()), + ) + .await?; + let physical = inner.get_part_unchunked(digest, 0, None).await?; + assert!( + physical.len() < poorly.len(), + "re-compression must shrink the stored stream ({} !< {})", + physical.len(), + poorly.len() + ); + assert_eq!( + zstd::stream::decode_all(&physical[..]).unwrap(), + data, + "re-compressed stream must still decode to the original content" + ); + + // Control: an already-well-compressed upload is kept as-is (never enlarged). + let (zstd2, _store2, inner2) = build(&spec).await?; + let well = zstd::bulk::compress(&data, 19).unwrap(); + zstd2 + .update_zstd_oneshot(digest, DigestHasherFunc::Sha256, Bytes::from(well.clone())) + .await?; + let physical2 = inner2.get_part_unchunked(digest, 0, None).await?; + assert_eq!( + &physical2[..], + &well[..], + "a well-compressed upload must be kept byte-for-byte, not enlarged" + ); + Ok(()) +} + +#[nativelink_test] +async fn update_zstd_skips_recompression_above_threshold() -> Result<(), Error> { + let data = compressible_data(64 * 1024); + let digest = digest_for(&data); + + let mut spec = spec_for(make_temp_path("zstd-above-threshold")); + spec.compression_level = Some(19); + spec.max_recompression_size = 16; // Uncompressed size far exceeds this. + let (zstd, _store, inner) = build(&spec).await?; + + let poorly = zstd::bulk::compress(&data, 1).unwrap(); + zstd.update_zstd_oneshot( + digest, + DigestHasherFunc::Sha256, + Bytes::from(poorly.clone()), + ) + .await?; + let physical = inner.get_part_unchunked(digest, 0, None).await?; + assert_eq!( + &physical[..], + &poorly[..], + "above max_recompression_size the stored stream must equal the upload byte-for-byte" + ); + Ok(()) +} + +#[nativelink_test] +async fn get_for_batch_selects_zstd_or_raw() -> Result<(), Error> { + let (zstd, store, _inner) = build(&spec()).await?; + let wire_store = Store::new(zstd.clone()) + .wire_compression_store() + .expect("ZstdStore must expose the wire-compression capability"); + + // Compressible blob (stored via the identity path so inner holds zstd). + let compressible = vec![0u8; 4096]; + let comp_digest = digest_for(&compressible); + store + .update_oneshot(comp_digest, compressible.clone().into()) + .await?; + + let (payload, compressor) = wire_store + .clone() + .get_for_batch(comp_digest, &[WireCompressor::Zstd]) + .await?; + assert_eq!( + compressor, + Some(WireCompressor::Zstd), + "compressible blob with accepts_zstd must return zstd" + ); + assert!( + (payload.len() as u64) < comp_digest.size_bytes(), + "physical zstd must be smaller than the uncompressed size" + ); + assert_eq!( + zstd::stream::decode_all(&payload[..]).unwrap(), + compressible, + "returned zstd must decode to the original content" + ); + + // Same blob, but the client does not accept zstd => raw. + let (raw, compressor) = wire_store.clone().get_for_batch(comp_digest, &[]).await?; + assert_eq!(compressor, None); + assert_eq!( + &raw[..], + &compressible[..], + "must return raw when zstd not accepted" + ); + + // Incompressible blob: physical zstd is not smaller => always raw. + let incompressible = random_bytes(2048); + let inc_digest = digest_for(&incompressible); + store + .update_oneshot(inc_digest, incompressible.clone().into()) + .await?; + let (raw, compressor) = wire_store + .get_for_batch(inc_digest, &[WireCompressor::Zstd]) + .await?; + assert_eq!( + compressor, None, + "incompressible blob must not be served as zstd" + ); + assert_eq!(&raw[..], &incompressible[..]); + Ok(()) +} + +#[nativelink_test] +async fn get_for_batch_decodes_concatenated_frames() -> Result<(), Error> { + // Regression for `get_for_batch`'s raw-decode path: the physical stream + // stored for a blob can be two independently-compressed zstd frames + // concatenated together (see `get_zstd_preserves_concatenated_frames`). + // `zstd::bulk::decompress` must handle that multi-frame input, not just a + // single-frame stream, when the client does not accept zstd. + const PART_A: &[u8] = b"first frame content aaaaaaaaaaaaaaaaaaaaaaaa"; + const PART_B: &[u8] = b"second frame content bbbbbbbbbbbbbbbbbbbbbbbb"; + + let frame_a = zstd::bulk::compress(PART_A, 3).unwrap(); + let frame_b = zstd::bulk::compress(PART_B, 3).unwrap(); + + let mut raw = PART_A.to_vec(); + raw.extend_from_slice(PART_B); + let digest = digest_for(&raw); + + let (zstd, _store, _inner) = build(&spec()).await?; + zstd.update_zstd( + digest, + DigestHasherFunc::Sha256, + reader_from(vec![Bytes::from(frame_a), Bytes::from(frame_b)]), + ) + .await?; + + // Client does not accept zstd => `get_for_batch` must decode the stored + // (concatenated-frame) physical bytes to the full raw content. + let (payload, is_zstd) = zstd.get_for_batch(digest, false).await?; + assert!( + !is_zstd, + "client that does not accept zstd must get raw bytes" + ); + assert_eq!( + &payload[..], + &raw[..], + "get_for_batch must decode a concatenated-frame physical stream to the full raw content" + ); + Ok(()) +} + +#[nativelink_test] +async fn zero_digest_zstd_fast_path() -> Result<(), Error> { + let recording = Arc::new(RecordingStore::default()); + std::fs::create_dir_all(TEMP_PATH).map_err(|e| make_err!(Code::Internal, "temp dir: {e}"))?; + let zstd = ZstdStore::new(&spec(), Store::new(recording.clone()))?; + + let zero = ZERO_BYTE_DIGESTS[0]; + + // get_zstd yields a valid zstd stream that decodes to empty. + let got = collect_zstd(&zstd, zero).await?; + assert!( + !got.is_empty(), + "zero-digest get_zstd must emit a real zstd frame" + ); + assert_eq!( + zstd::stream::decode_all(&got[..]).unwrap().len(), + 0, + "zero-digest zstd stream must decode to empty" + ); + + // update_zstd of an empty-decoding stream succeeds without touching inner. + let empty = zstd::bulk::compress(&[], 3).unwrap(); + let wire = zstd + .update_zstd_oneshot(zero, DigestHasherFunc::Sha256, Bytes::from(empty.clone())) + .await?; + assert_eq!(wire, empty.len() as u64); + assert_eq!( + recording.update_count.load(Ordering::Relaxed), + 0, + "zero digest must never reach the inner store" + ); + + // A stream that decodes to non-empty content is rejected for a zero digest. + let non_empty = zstd::bulk::compress(b"not empty", 3).unwrap(); + let err = zstd + .update_zstd_oneshot(zero, DigestHasherFunc::Sha256, Bytes::from(non_empty)) + .await + .expect_err("non-empty content under a zero digest must be rejected"); + assert_eq!(err.code, Code::InvalidArgument); + Ok(()) +} + +#[nativelink_test] +async fn cancelled_update_zstd_leaves_no_temp_files() -> Result<(), Error> { + let temp = make_temp_path("zstd-cancel"); + let (zstd, _store, _inner) = build(&spec_for(temp.clone())).await?; + + let data = compressible_data(256 * 1024); + let compressed = zstd::bulk::compress(&data, 3).unwrap(); + let digest = digest_for(&data); + + let (mut tx, rx) = make_buf_channel_pair(); + let zstd_clone = zstd.clone(); + // `spawn!` returns a guard that aborts the task when it is dropped. + let handle = spawn!("zstd_test_cancel", async move { + zstd_clone + .update_zstd(digest, DigestHasherFunc::Sha256, rx) + .await + }); + + // Send only the first half of the compressed stream, never EOF. + let half = compressed.len() / 2; + tx.send(Bytes::copy_from_slice(&compressed[..half])) + .await + .ok(); + tokio::time::sleep(Duration::from_millis(50)).await; + + // Cancel the upload mid-stream (dropping the guard aborts it), then + // release the sender. + drop(handle); + drop(tx); + tokio::time::sleep(Duration::from_millis(150)).await; + + assert_eq!( + dir_entry_count(&temp), + 0, + "a cancelled upload must not leave staged temp files" + ); + Ok(()) +} + +#[nativelink_test] +async fn concurrent_uploads_respect_staging_semaphore() -> Result<(), Error> { + const DATA_A: &[u8] = b"concurrent upload alpha aaaaaaaaaaaaaaaaaaaaaaaaaaaa"; + const DATA_B: &[u8] = b"concurrent upload beta bbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"; + + let mut spec = spec_for(make_temp_path("zstd-semaphore")); + spec.max_concurrent_staged_uploads = 1; // Serialize staging. + let (zstd, store, _inner) = build(&spec).await?; + + let digest_a = digest_for(DATA_A); + let digest_b = digest_for(DATA_B); + let comp_a = zstd::bulk::compress(DATA_A, 3).unwrap(); + let comp_b = zstd::bulk::compress(DATA_B, 3).unwrap(); + + let za = zstd.clone(); + let zb = zstd.clone(); + let (ra, rb) = tokio::join!( + za.update_zstd_oneshot(digest_a, DigestHasherFunc::Sha256, Bytes::from(comp_a)), + zb.update_zstd_oneshot(digest_b, DigestHasherFunc::Sha256, Bytes::from(comp_b)), + ); + ra?; + rb?; + + assert_eq!( + &store.get_part_unchunked(digest_a, 0, None).await?[..], + DATA_A, + "first concurrent upload must round-trip" + ); + assert_eq!( + &store.get_part_unchunked(digest_b, 0, None).await?[..], + DATA_B, + "second concurrent upload must round-trip" + ); + Ok(()) +} + +// --------------------------------------------------------------------------- +// Cross-wrapper integration: ZstdStore composed with other stores. +// +// These prove ZstdStore is a well-behaved participant in a store stack. Every +// test does a full identity round-trip (write raw via `update_oneshot`, read raw +// back via `get_part_unchunked`) through the composed stack. Where a wrapper +// lives INSIDE ZstdStore (so it observes the physical zstd stream) the test also +// asserts that wrapper's side effect still fires. +// +// Identity-path tests do not touch `temp_path` (only the zstd fast path stages +// files), so they reuse the shared `spec()` without creating a staging dir. +// --------------------------------------------------------------------------- + +// 1. ZstdStore over `fast_slow` (fast=memory, slow=memory): round-trip, then +// prove a slow-hit read repopulates the fast tier with the physical zstd blob. +#[nativelink_test] +async fn over_fast_slow_populates_fast_tier() -> Result<(), Error> { + let fast_mem = MemoryStore::new(&MemorySpec::default()); + let slow_mem = MemoryStore::new(&MemorySpec::default()); + let fast_slow = Store::new(FastSlowStore::new( + &FastSlowSpec { + fast: StoreSpec::Memory(MemorySpec::default()), + slow: StoreSpec::Memory(MemorySpec::default()), + fast_direction: StoreDirection::default(), + slow_direction: StoreDirection::default(), + bypass_dedup_threshold_bytes: 0, + }, + Store::new(fast_mem.clone()), + Store::new(slow_mem.clone()), + )); + let store = Store::new(ZstdStore::new(&spec(), fast_slow)?); + + let data = compressible_data(64 * 1024); + let digest = digest_for(&data); + store.update_oneshot(digest, data.clone().into()).await?; + assert_eq!( + &store.get_part_unchunked(digest, 0, None).await?[..], + &data[..], + "round-trip through zstd-over-fast_slow must return the raw bytes" + ); + + // The physical (compressed) bytes landed in both tiers; capture the slow copy. + let physical = slow_mem.get_part_unchunked(digest, 0, None).await?; + assert!( + physical.len() < data.len(), + "inner store must hold the compressed physical stream ({} !< {})", + physical.len(), + data.len() + ); + + // Evict the fast tier, leaving only the slow tier populated. + assert!( + fast_mem.remove_entry(digest.into()).await, + "fast tier should have held the blob before eviction" + ); + assert_eq!( + fast_mem.has(digest).await?, + None, + "fast tier must be empty after eviction" + ); + + // A read through the stack is a slow-hit that must repopulate the fast tier. + assert_eq!( + &store.get_part_unchunked(digest, 0, None).await?[..], + &data[..], + "slow-hit read-through must still return the raw bytes" + ); + assert_eq!( + fast_mem.has(digest).await?, + Some(physical.len() as u64), + "fast tier must be repopulated with the physical zstd blob" + ); + assert_eq!( + &fast_mem.get_part_unchunked(digest, 0, None).await?[..], + &physical[..], + "the repopulated fast-tier bytes must be the physical zstd stream" + ); + Ok(()) +} + +// 2. ZstdStore over `dedup`: a blob whose physical stream exceeds the dedup block +// size must split across multiple content chunks yet round-trip byte-for-byte. +#[nativelink_test] +async fn over_dedup_splits_and_round_trips() -> Result<(), Error> { + let content_mem = MemoryStore::new(&MemorySpec::default()); + let dedup = Store::new(DedupStore::new( + &DedupSpec { + index_store: StoreSpec::Memory(MemorySpec::default()), + content_store: StoreSpec::Memory(MemorySpec::default()), + min_size: 8 * 1024, + normal_size: 32 * 1024, + max_size: 128 * 1024, + max_concurrent_fetch_per_get: 10, + }, + Store::new(MemoryStore::new(&MemorySpec::default())), + Store::new(content_mem.clone()), + )?); + let store = Store::new(ZstdStore::new(&spec(), dedup)?); + + // Incompressible data so the physical zstd stream stays ~256 KiB, comfortably + // above the 128 KiB max block size and therefore split into several chunks. + let data = random_bytes(256 * 1024); + let digest = digest_for(&data); + store.update_oneshot(digest, data.clone().into()).await?; + assert_eq!( + &store.get_part_unchunked(digest, 0, None).await?[..], + &data[..], + "round-trip through zstd-over-dedup must return the raw bytes" + ); + assert!( + content_mem.len_for_test() > 1, + "dedup must split the physical zstd stream into multiple content chunks, got {}", + content_mem.len_for_test() + ); + Ok(()) +} + +// 3. ZstdStore over `existence_cache`: round-trip and prove the existence cache +// side effect (population on write) still fires for the forwarded digest. +#[nativelink_test] +async fn over_existence_cache_fires_side_effect() -> Result<(), Error> { + let inner = Store::new(MemoryStore::new(&MemorySpec::default())); + let existence = ExistenceCacheStore::new( + &ExistenceCacheSpec { + backend: StoreSpec::Noop(NoopSpec::default()), // Unused: inner is passed directly. + eviction_policy: None, + }, + inner, + ); + let store = Store::new(ZstdStore::new(&spec(), Store::new(existence.clone()))?); + + let data = compressible_data(4096); + let digest = digest_for(&data); + assert!( + !existence.exists_in_cache(&digest).await, + "digest must not be cached before the write" + ); + + store.update_oneshot(digest, data.clone().into()).await?; + assert!( + existence.exists_in_cache(&digest).await, + "write through zstd must populate the inner existence cache" + ); + assert_eq!( + &store.get_part_unchunked(digest, 0, None).await?[..], + &data[..], + "round-trip through zstd-over-existence_cache must return the raw bytes" + ); + // `has` is served from the now-populated existence cache and reports the + // uncompressed digest size (not the physical zstd size). + assert_eq!( + store.has(digest).await?, + Some(data.len() as u64), + "has must report the uncompressed size via the existence cache" + ); + Ok(()) +} + +// 4. ZstdStore over `cache_metrics`: round-trip through the metrics wrapper. The +// metrics themselves are process-global OpenTelemetry counters (not unit +// assertable without a metrics-reader harness, mirroring the existing +// `cache_metrics_store_test.rs`), so we assert that operations route through +// the wrapper and that zstd compression is still applied behind it. +#[nativelink_test] +async fn over_cache_metrics_round_trips() -> Result<(), Error> { + let inner = Store::new(MemoryStore::new(&MemorySpec::default())); + let metrics = CacheMetricsStore::new( + &CacheMetricsSpec { + cache_type: "cas".to_string(), + backend: StoreSpec::Memory(MemorySpec::default()), // Unused: inner is passed directly. + }, + inner.clone(), + ); + let store = Store::new(ZstdStore::new(&spec(), Store::new(metrics))?); + + let data = compressible_data(4096); + let digest = digest_for(&data); + store.update_oneshot(digest, data.clone().into()).await?; + assert_eq!( + &store.get_part_unchunked(digest, 0, None).await?[..], + &data[..], + "round-trip through zstd-over-cache_metrics must return the raw bytes" + ); + assert_eq!(store.has(digest).await?, Some(data.len() as u64)); + let physical = inner.get_part_unchunked(digest, 0, None).await?; + assert!( + physical.len() < data.len(), + "zstd compression must still be applied behind the metrics wrapper" + ); + Ok(()) +} + +// 5a. ZstdStore over `size_partitioning`: identity round-trip; the physical zstd +// stream is routed by the (uncompressed) digest size. +#[nativelink_test] +async fn over_size_partitioning_round_trips() -> Result<(), Error> { + let lower = MemoryStore::new(&MemorySpec::default()); + let upper = MemoryStore::new(&MemorySpec::default()); + let size_part = Store::new(SizePartitioningStore::new( + &SizePartitioningSpec { + size: 100, + lower_store: StoreSpec::Memory(MemorySpec::default()), + upper_store: StoreSpec::Memory(MemorySpec::default()), + }, + Store::new(lower.clone()), + Store::new(upper.clone()), + )); + let store = Store::new(ZstdStore::new(&spec(), size_part)?); + + // Uncompressed size (4096) >= partition threshold (100) => routed to `upper`. + let data = compressible_data(4096); + let digest = digest_for(&data); + store.update_oneshot(digest, data.clone().into()).await?; + assert_eq!( + &store.get_part_unchunked(digest, 0, None).await?[..], + &data[..], + "round-trip through zstd-over-size_partitioning must return the raw bytes" + ); + assert!( + upper.has(digest).await?.is_some(), + "a blob above the partition threshold must be stored in the upper partition" + ); + assert_eq!( + lower.has(digest).await?, + None, + "the lower partition must not hold the blob" + ); + Ok(()) +} + +// 5b. ZstdStore over `shard`: identity round-trip through a two-way shard. +#[nativelink_test] +async fn over_shard_round_trips() -> Result<(), Error> { + let shard = Store::new(ShardStore::new( + &ShardSpec { + stores: vec![ + ShardConfig { + store: StoreSpec::Memory(MemorySpec::default()), + weight: Some(1), + }, + ShardConfig { + store: StoreSpec::Memory(MemorySpec::default()), + weight: Some(1), + }, + ], + }, + vec![ + Store::new(MemoryStore::new(&MemorySpec::default())), + Store::new(MemoryStore::new(&MemorySpec::default())), + ], + )?); + let store = Store::new(ZstdStore::new(&spec(), shard)?); + + let data = compressible_data(4096); + let digest = digest_for(&data); + store.update_oneshot(digest, data.clone().into()).await?; + assert_eq!( + &store.get_part_unchunked(digest, 0, None).await?[..], + &data[..], + "round-trip through zstd-over-shard must return the raw bytes" + ); + assert_eq!(store.has(digest).await?, Some(data.len() as u64)); + Ok(()) +} + +// 5c. ZstdStore over `ref`: identity round-trip through a ref store that resolves +// to a named memory backend via the StoreManager. +#[nativelink_test] +async fn over_ref_round_trips() -> Result<(), Error> { + let store_manager = Arc::new(StoreManager::new()); + let backing = Store::new(MemoryStore::new(&MemorySpec::default())); + store_manager.add_store("backing", backing.clone())?; + let ref_store = Store::new(RefStore::new( + &RefSpec { + name: "backing".to_string(), + }, + Arc::downgrade(&store_manager), + )); + store_manager.add_store("ref", ref_store.clone())?; + store_manager.run_post_init().await.unwrap(); + + let store = Store::new(ZstdStore::new(&spec(), ref_store)?); + + let data = compressible_data(4096); + let digest = digest_for(&data); + store.update_oneshot(digest, data.clone().into()).await?; + assert_eq!( + &store.get_part_unchunked(digest, 0, None).await?[..], + &data[..], + "round-trip through zstd-over-ref must return the raw bytes" + ); + // The resolved backing store must physically hold the compressed stream. + let physical = backing.get_part_unchunked(digest, 0, None).await?; + assert!( + physical.len() < data.len(), + "the ref-resolved backend must hold the compressed physical stream" + ); + Ok(()) +} + +// 6. A wrapper OUTSIDE ZstdStore (cache_metrics -> zstd_store -> memory). This +// exercises only the StoreDriver identity path (the service zstd fast path is +// unreachable through an outer wrapper), proving that placing a wrapper +// outside ZstdStore preserves identity correctness and does not bypass the +// outer wrapper's operations while zstd compression stays active underneath. +#[nativelink_test] +async fn wrapper_outside_zstd_preserves_identity() -> Result<(), Error> { + let backing = Store::new(MemoryStore::new(&MemorySpec::default())); + let zstd = Store::new(ZstdStore::new(&spec(), backing.clone())?); + let outer = Store::new(CacheMetricsStore::new( + &CacheMetricsSpec { + cache_type: "cas".to_string(), + backend: StoreSpec::Memory(MemorySpec::default()), // Unused: zstd is passed directly. + }, + zstd, + )); + + let data = compressible_data(4096); + let digest = digest_for(&data); + outer.update_oneshot(digest, data.clone().into()).await?; + assert_eq!( + &outer.get_part_unchunked(digest, 0, None).await?[..], + &data[..], + "identity round-trip through an outer wrapper must return the raw bytes" + ); + assert_eq!( + outer.has(digest).await?, + Some(data.len() as u64), + "the outer wrapper must report the uncompressed size" + ); + let physical = backing.get_part_unchunked(digest, 0, None).await?; + assert!( + physical.len() < data.len(), + "zstd compression must still be active behind the outer wrapper" + ); + Ok(()) +} + +// 7. Concatenated-frame integrity through a wrapper: upload two concatenated zstd +// frames (fast path) into a ZstdStore-over-fast_slow, read back via `get_zstd` +// byte-for-byte, and via the identity `get_part` decode to the full content. +#[nativelink_test] +async fn concatenated_frames_through_fast_slow() -> Result<(), Error> { + const PART_A: &[u8] = b"first frame content aaaaaaaaaaaaaaaaaaaaaaaa"; + const PART_B: &[u8] = b"second frame content bbbbbbbbbbbbbbbbbbbbbbbb"; + + let frame_a = zstd::bulk::compress(PART_A, 3).unwrap(); + let frame_b = zstd::bulk::compress(PART_B, 3).unwrap(); + let mut concatenated = frame_a.clone(); + concatenated.extend_from_slice(&frame_b); + + let mut raw = PART_A.to_vec(); + raw.extend_from_slice(PART_B); + let digest = digest_for(&raw); + + // The fast path stages files, so a real writable temp dir is required. + let temp = make_temp_path("zstd-concat-fast-slow"); + std::fs::create_dir_all(&temp) + .map_err(|e| make_err!(Code::Internal, "Failed to create test temp dir: {e}"))?; + let fast_slow = Store::new(FastSlowStore::new( + &FastSlowSpec { + fast: StoreSpec::Memory(MemorySpec::default()), + slow: StoreSpec::Memory(MemorySpec::default()), + fast_direction: StoreDirection::default(), + slow_direction: StoreDirection::default(), + bypass_dedup_threshold_bytes: 0, + }, + Store::new(MemoryStore::new(&MemorySpec::default())), + Store::new(MemoryStore::new(&MemorySpec::default())), + )); + let zstd = ZstdStore::new(&spec_for(temp), fast_slow)?; + let store = Store::new(zstd.clone()); + + let wire = zstd + .update_zstd( + digest, + DigestHasherFunc::Sha256, + reader_from(vec![Bytes::from(frame_a), Bytes::from(frame_b)]), + ) + .await?; + assert_eq!(wire, concatenated.len() as u64); + + let got = collect_zstd(&zstd, digest).await?; + assert_eq!( + &got[..], + &concatenated[..], + "passthrough through fast_slow must preserve the exact concatenated-frame bytes" + ); + + // The identity view decodes the concatenated frames to the full content. + let decoded = store.get_part_unchunked(digest, 0, None).await?; + assert_eq!( + &decoded[..], + &raw[..], + "the identity read must decode the concatenated frames to the full raw content" + ); + Ok(()) +} + +// 8. Rollout from an empty namespace: a fresh memory backend, a full write->read +// cycle, confirming the digest is absent before the write and present after. +#[nativelink_test] +async fn rollout_from_empty_namespace() -> Result<(), Error> { + const DATA: &[u8] = b"rollout payload from an empty dedicated namespace aaaaaaaaaa"; + + let inner = Store::new(MemoryStore::new(&MemorySpec::default())); + let store = Store::new(ZstdStore::new(&spec(), inner.clone())?); + let digest = digest_for(DATA); + + assert_eq!( + store.has(digest).await?, + None, + "an empty namespace must report the digest as absent" + ); + assert_eq!( + inner.has(digest).await?, + None, + "the inner backend must start empty" + ); + + store.update_oneshot(digest, DATA.into()).await?; + assert_eq!( + &store.get_part_unchunked(digest, 0, None).await?[..], + DATA, + "the rollout write->read cycle must return the raw bytes" + ); + assert_eq!( + store.has(digest).await?, + Some(DATA.len() as u64), + "after the write the digest must be present at its uncompressed size" + ); + Ok(()) +} + +// --------------------------------------------------------------------------- +// Security & starvation hardening (bounded decode, descriptor pinning, commit +// deadline). See the PR's issues 1-3. +// --------------------------------------------------------------------------- + +/// Inner store that, on `update_with_whole_file`, first *replaces* the staged +/// pathname's contents (simulating an observe-and-replace attacker) and then +/// reads the retained descriptor it was handed, storing whatever the descriptor +/// yields into an inner `MemoryStore`. If the zstd store commits from the +/// validated descriptor (as it must), the stored bytes are the validated stream +/// regardless of what happened to the pathname. +#[derive(MetricsComponent)] +struct ClobberingStore { + #[metric(group = "inner")] + inner: Store, +} + +#[async_trait] +impl StoreDriver for ClobberingStore { + async fn post_init(self: Arc) -> Result<(), Error> { + Ok(()) + } + + async fn has_with_results( + self: Pin<&Self>, + keys: &[StoreKey<'_>], + results: &mut [Option], + ) -> Result<(), Error> { + self.inner.has_with_results(keys, results).await + } + + async fn update( + self: Pin<&Self>, + key: StoreKey<'_>, + reader: DropCloserReadHalf, + size_info: UploadSizeInfo, + ) -> Result { + self.inner + .as_store_driver_pin() + .update(key, reader, size_info) + .await + } + + async fn update_with_whole_file( + self: Pin<&Self>, + key: StoreKey<'_>, + path: OsString, + mut file: FileSlot, + _upload_size: UploadSizeInfo, + ) -> Result<(u64, Option), Error> { + // Replace the pathname with attacker-controlled content *after* the zstd + // store validated the descriptor. Unlink first so the retained fd points + // at a now-orphaned inode, then create a fresh file at the same path. + drop(std::fs::remove_file(&path)); + std::fs::write(&path, b"CLOBBERED-BY-ATTACKER").expect("clobber write"); + + // Read the *descriptor* the store handed us (the validated bytes). + file.rewind() + .await + .map_err(|e| make_err!(Code::Internal, "rewind clobber fd: {e}"))?; + let mut buf = Vec::new(); + file.read_to_end(&mut buf) + .await + .map_err(|e| make_err!(Code::Internal, "read clobber fd: {e}"))?; + let size = buf.len() as u64; + self.inner + .update_oneshot(key.into_digest(), Bytes::from(buf)) + .await?; + Ok((size, None)) + } + + async fn get_part( + self: Pin<&Self>, + key: StoreKey<'_>, + writer: &mut DropCloserWriteHalf, + offset: u64, + length: Option, + ) -> Result<(), Error> { + self.inner + .as_store_driver_pin() + .get_part(key, writer, offset, length) + .await + } + + fn inner_store(&self, _key: Option) -> &dyn StoreDriver { + self + } + + fn as_any(&self) -> &(dyn core::any::Any + Sync + Send + 'static) { + self + } + + fn as_any_arc(self: Arc) -> Arc { + self + } + + fn register_remove_callback(self: Arc, _callback: RemoveCallback) -> Result<(), Error> { + Ok(()) + } +} + +default_health_status_indicator!(ClobberingStore); + +/// Inner store whose whole-file commit never resolves, used to prove the zstd +/// store bounds a stalled backend commit with its `commit_timeout` and releases +/// the staging permit afterwards. +#[derive(Default, MetricsComponent)] +struct StallStore {} + +#[async_trait] +impl StoreDriver for StallStore { + async fn post_init(self: Arc) -> Result<(), Error> { + Ok(()) + } + + async fn has_with_results( + self: Pin<&Self>, + _keys: &[StoreKey<'_>], + _results: &mut [Option], + ) -> Result<(), Error> { + Ok(()) + } + + async fn update( + self: Pin<&Self>, + _key: StoreKey<'_>, + _reader: DropCloserReadHalf, + _size_info: UploadSizeInfo, + ) -> Result { + // Never completes. + core::future::pending::<()>().await; + unreachable!("StallStore::update never resolves") + } + + async fn update_with_whole_file( + self: Pin<&Self>, + _key: StoreKey<'_>, + _path: OsString, + _file: FileSlot, + _upload_size: UploadSizeInfo, + ) -> Result<(u64, Option), Error> { + // Simulate a permanently stalled backend commit. + core::future::pending::<()>().await; + unreachable!("StallStore::update_with_whole_file never resolves") + } + + async fn get_part( + self: Pin<&Self>, + _key: StoreKey<'_>, + _writer: &mut DropCloserWriteHalf, + _offset: u64, + _length: Option, + ) -> Result<(), Error> { + Err(make_err!(Code::NotFound, "Not found")) + } + + fn inner_store(&self, _key: Option) -> &dyn StoreDriver { + self + } + + fn as_any(&self) -> &(dyn core::any::Any + Sync + Send + 'static) { + self + } + + fn as_any_arc(self: Arc) -> Arc { + self + } + + fn register_remove_callback(self: Arc, _callback: RemoveCallback) -> Result<(), Error> { + Ok(()) + } +} + +default_health_status_indicator!(StallStore); + +/// A compressed stream that decodes to non-empty content is rejected under a +/// zero digest — via both the streaming and one-shot entry points — and stops +/// at the output sink without materializing the (large) decoded output. +#[nativelink_test] +async fn zero_digest_decoding_nonempty_is_rejected_at_sink() -> Result<(), Error> { + let temp = make_temp_path("zstd-zero-bomb"); + let (zstd, _store, _inner) = build(&spec_for(temp.clone())).await?; + + // 16 MiB of zeros compresses to a few KiB. Under a zero digest the decoded + // output cap is 0, so the first decoded block is rejected immediately — the + // 16 MiB is never materialized (the test would be far slower / OOM if it + // were). + let bomb = zstd::bulk::compress(&vec![0u8; 16 * 1024 * 1024], 3).unwrap(); + assert!( + bomb.len() < 64 * 1024, + "bomb should be tiny compressed ({} bytes)", + bomb.len() + ); + let zero = ZERO_BYTE_DIGESTS[0]; + + // One-shot entry point. + let err = zstd + .update_zstd_oneshot(zero, DigestHasherFunc::Sha256, Bytes::from(bomb.clone())) + .await + .expect_err("a non-empty-decoding zero-digest upload must be rejected"); + assert_eq!(err.code, Code::InvalidArgument, "got: {err}"); + + // Streaming entry point. + let err = zstd + .update_zstd( + zero, + DigestHasherFunc::Sha256, + reader_from(vec![Bytes::from(bomb)]), + ) + .await + .expect_err("a non-empty-decoding zero-digest stream must be rejected"); + assert_eq!(err.code, Code::InvalidArgument, "got: {err}"); + + assert_eq!( + dir_entry_count(&temp), + 0, + "zero-digest validation must never stage a temp file" + ); + Ok(()) +} + +/// A non-zero upload whose decoded output exceeds the declared digest size is +/// stopped at the decoder sink (`InvalidArgument`), commits nothing, and leaves +/// no staging file. The decoded output (256 KiB) dwarfs the declared size (100 +/// bytes), so the sink rejects the very first over-limit block. +#[nativelink_test] +async fn decoded_output_exceeding_digest_size_is_stopped_at_sink() -> Result<(), Error> { + let temp = make_temp_path("zstd-decode-overflow"); + let (zstd, _store, inner) = build(&spec_for(temp.clone())).await?; + + let data = compressible_data(256 * 1024); + let compressed = zstd::bulk::compress(&data, 3).unwrap(); + // Claim a digest for only the first 100 decoded bytes: the actual decoded + // stream is far larger than the declared size. + let bad_digest = digest_for(&data[..100]); + assert_eq!(bad_digest.size_bytes(), 100); + + let err = zstd + .update_zstd_oneshot( + bad_digest, + DigestHasherFunc::Sha256, + Bytes::from(compressed), + ) + .await + .expect_err("decoded output larger than the digest size must be rejected"); + assert_eq!(err.code, Code::InvalidArgument, "got: {err}"); + + assert_eq!( + inner.has(bad_digest).await?, + None, + "an over-decoding upload must not commit to the inner store" + ); + assert_eq!( + dir_entry_count(&temp), + 0, + "an over-decoding upload must not leave staged temp files" + ); + Ok(()) +} + +/// The commit streams the exact validated descriptor, not a reopened pathname: +/// replacing the pathname's contents after validation cannot change what is +/// committed. +#[nativelink_test] +async fn commit_uses_validated_descriptor_not_reopened_path() -> Result<(), Error> { + const DATA: &[u8] = b"descriptor-pinned commit payload aaaaaaaaaaaaaaaaaaaaaaaa"; + + let temp = make_temp_path("zstd-fd-pin"); + std::fs::create_dir_all(&temp).map_err(|e| make_err!(Code::Internal, "temp dir: {e}"))?; + let inner_mem = Store::new(MemoryStore::new(&MemorySpec::default())); + let clobber = Store::new(Arc::new(ClobberingStore { + inner: inner_mem.clone(), + })); + let zstd = ZstdStore::new(&spec_for(temp.clone()), clobber)?; + + let compressed = zstd::bulk::compress(DATA, 3).unwrap(); + let digest = digest_for(DATA); + + zstd.update_zstd_oneshot( + digest, + DigestHasherFunc::Sha256, + Bytes::from(compressed.clone()), + ) + .await?; + + // The bytes the inner store committed came from the retained descriptor, so + // they equal the validated stream — NOT the "CLOBBERED-BY-ATTACKER" content + // that replaced the pathname during the commit. + let committed = inner_mem.get_part_unchunked(digest, 0, None).await?; + assert_eq!( + &committed[..], + &compressed[..], + "committed bytes must be the validated descriptor's contents, not the replaced pathname's" + ); + assert_eq!( + zstd::stream::decode_all(&committed[..]).unwrap(), + DATA, + "the descriptor-committed stream must still decode to the original content" + ); + assert_eq!( + dir_entry_count(&temp), + 0, + "a successful commit must leave no staged temp files" + ); + Ok(()) +} + +/// A permanently stalled backend commit is bounded by `commit_timeout`: the +/// upload fails with `DeadlineExceeded`, the staged file is removed, and the +/// staging permit is released — so a second upload behind a `max_concurrent +/// staged_uploads = 1` bound is admitted rather than blocked forever. +// Uses a short real-time `commit_timeout_s = 1` rather than a paused clock: +// the `nativelink-store` test crate does not enable tokio's `test-util` +// feature, so `start_paused` is unavailable here. +#[nativelink_test] +async fn stalled_commit_times_out_and_releases_staging_slot() -> Result<(), Error> { + const DATA: &[u8] = b"payload whose commit stalls forever bbbbbbbbbbbbbbbbbbbb"; + + let temp = make_temp_path("zstd-commit-stall"); + let mut spec = spec_for(temp.clone()); + spec.max_concurrent_staged_uploads = 1; + spec.commit_timeout_s = 1; + std::fs::create_dir_all(&temp).map_err(|e| make_err!(Code::Internal, "temp dir: {e}"))?; + let zstd = ZstdStore::new(&spec, Store::new(Arc::new(StallStore {})))?; + + let compressed = zstd::bulk::compress(DATA, 3).unwrap(); + let digest = digest_for(DATA); + + let err = zstd + .update_zstd_oneshot( + digest, + DigestHasherFunc::Sha256, + Bytes::from(compressed.clone()), + ) + .await + .expect_err("a stalled commit must fail rather than hang"); + assert_eq!( + err.code, + Code::DeadlineExceeded, + "a stalled commit must surface DeadlineExceeded, got: {err}" + ); + assert_eq!( + dir_entry_count(&temp), + 0, + "a timed-out commit must remove its staged temp file" + ); + + // If the first upload had not released the staging permit, this second + // upload (staging bound = 1) would block on the semaphore forever and the + // test would hang. It resolving at all proves the slot was freed. + let err2 = zstd + .update_zstd_oneshot(digest, DigestHasherFunc::Sha256, Bytes::from(compressed)) + .await + .expect_err("second stalled commit must also time out, not hang"); + assert_eq!(err2.code, Code::DeadlineExceeded, "got: {err2}"); + assert_eq!( + dir_entry_count(&temp), + 0, + "the second timed-out commit must also clean up" + ); + Ok(()) +} + +// --------------------------------------------------------------------------- +// Inline commit path (small compressed oneshot payloads) +// --------------------------------------------------------------------------- + +/// A compressed oneshot payload at or below `max_inline_commit_size` is +/// validated and committed straight from memory: it round-trips correctly and +/// never creates a staging file. +#[nativelink_test] +async fn inline_commit_skips_the_staging_file() -> Result<(), Error> { + const DATA: &[u8] = b"batch-sized payload committed without touching disk aaaaaaaa"; + + let temp = make_temp_path("zstd-inline"); + let spec = spec_for_inline(temp.clone()); + let (zstd, store, inner) = build(&spec).await?; + let compressed = zstd::bulk::compress(DATA, 3).unwrap(); + let digest = digest_for(DATA); + assert!( + (compressed.len() as u64) <= 4 * 1024 * 1024, + "the payload must be inline-eligible for this test to mean anything" + ); + + zstd.update_zstd_oneshot( + digest, + DigestHasherFunc::Sha256, + Bytes::from(compressed.clone()), + ) + .await?; + + assert_eq!( + &inner.get_part_unchunked(digest, 0, None).await?[..], + &compressed[..], + "the inline commit must store the client's bytes verbatim" + ); + assert_eq!( + &store.get_part_unchunked(digest, 0, None).await?[..], + DATA, + "the inline-committed blob must still decode through the identity view" + ); + assert_eq!( + dir_entry_count(&temp), + 0, + "an inline commit must never create a staging file" + ); + Ok(()) +} + +/// The inline path validates against the declared digest just like staging does, +/// and rejects a mismatch with `InvalidArgument` without storing anything. +#[nativelink_test] +async fn inline_commit_rejects_hash_mismatch() -> Result<(), Error> { + let temp = make_temp_path("zstd-inline-mismatch"); + let (zstd, _store, inner) = build(&spec_for_inline(temp.clone())).await?; + + let compressed = zstd::bulk::compress(b"inline payload", 3).unwrap(); + let wrong_digest = digest_for(b"a completely different payload"); + + let err = zstd + .update_zstd_oneshot( + wrong_digest, + DigestHasherFunc::Sha256, + Bytes::from(compressed), + ) + .await + .expect_err("an inline upload whose digest does not match must be rejected"); + assert_eq!(err.code, Code::InvalidArgument, "got: {err}"); + assert!( + inner + .get_part_unchunked(wrong_digest, 0, None) + .await + .is_err(), + "a rejected inline upload must not be committed" + ); + assert_eq!(dir_entry_count(&temp), 0); + Ok(()) +} + +/// A oneshot payload larger than `max_inline_commit_size` is fed through the +/// streaming staging path. When staging rejects it, the reported error must be +/// staging's own `InvalidArgument` — not the `Internal` "failed to feed" error +/// the feeder task produces as a *consequence* of the reader being dropped. +#[nativelink_test] +async fn large_oneshot_reports_the_validation_error_not_the_feed_error() -> Result<(), Error> { + let temp = make_temp_path("zstd-oneshot-error-order"); + let mut spec = spec_for_inline(temp.clone()); + spec.max_inline_commit_size = 1; // Force the streaming staging path. + let (zstd, _store, inner) = build(&spec).await?; + + // Large enough that the feeder cannot hand the whole payload off before + // staging notices the mismatch and drops the reader. + let data = compressible_data(2 * 1024 * 1024); + let compressed = zstd::bulk::compress(&data, 3).unwrap(); + let wrong_digest = digest_for(b"not the uploaded content"); + + let err = zstd + .update_zstd_oneshot( + wrong_digest, + DigestHasherFunc::Sha256, + Bytes::from(compressed), + ) + .await + .expect_err("a digest mismatch must be rejected"); + assert_eq!( + err.code, + Code::InvalidArgument, + "the staging error must win over the feeder's channel error, got: {err}" + ); + assert!( + inner + .get_part_unchunked(wrong_digest, 0, None) + .await + .is_err(), + "a rejected upload must not be committed" + ); + assert_eq!(dir_entry_count(&temp), 0); + Ok(()) +} + +// --------------------------------------------------------------------------- +// Admission, deadlines, and config validation +// --------------------------------------------------------------------------- + +/// `max_recompression_size` re-encodes at `compression_level`, so a positive +/// ceiling with no level configured would silently do nothing. It is rejected at +/// construction instead. +#[nativelink_test] +async fn new_rejects_recompression_size_without_a_level() -> Result<(), Error> { + let mut spec = spec_for(make_temp_path("zstd-recompress-no-level")); + spec.max_recompression_size = 1024 * 1024; + spec.compression_level = None; + let inner = Store::new(MemoryStore::new(&MemorySpec::default())); + let err = ZstdStore::new(&spec, inner.clone()) + .expect_err("max_recompression_size without compression_level must be rejected"); + assert_eq!(err.code, Code::InvalidArgument, "got: {err}"); + + // With a level it constructs fine. + spec.compression_level = Some(9); + ZstdStore::new(&spec, inner)?; + Ok(()) +} + +/// A client that trickles bytes forever resets any per-message idle timeout, so +/// the store applies a *total* validate-and-stage deadline. On expiry the upload +/// fails with `DeadlineExceeded`, the staged file is removed, and the staging +/// slot is released — proven by a second upload behind a bound of 1 completing. +#[nativelink_test] +async fn stage_timeout_bounds_a_trickling_client_and_frees_the_slot() -> Result<(), Error> { + const DATA: &[u8] = b"payload delivered one byte at a time aaaaaaaaaaaaaaaaaaaa"; + + let temp = make_temp_path("zstd-stage-timeout"); + let mut spec = spec_for(temp.clone()); + spec.max_concurrent_staged_uploads = 1; + spec.stage_timeout_s = 1; + let (zstd, _store, _inner) = build(&spec).await?; + + let compressed = zstd::bulk::compress(DATA, 3).unwrap(); + let digest = digest_for(DATA); + + // A reader that hands over one byte every 100ms and never reaches EOF: every + // individual wait is short, but the upload never finishes. + let (mut tx, rx) = make_buf_channel_pair(); + let stop = Arc::new(AtomicBool::new(false)); + let trickle_stop = stop.clone(); + let trickle = Bytes::from(compressed.clone()); + background_spawn!("zstd_test_trickle", async move { + let mut index = 0usize; + while !trickle_stop.load(Ordering::Relaxed) { + let byte = trickle.slice(index % trickle.len()..).slice(..1); + if tx.send(byte).await.is_err() { + return; + } + index += 1; + tokio::time::sleep(Duration::from_millis(100)).await; + } + }); + + let err = zstd + .update_zstd(digest, DigestHasherFunc::Sha256, rx) + .await + .expect_err("a trickling upload must be bounded by the staging deadline"); + assert_eq!( + err.code, + Code::DeadlineExceeded, + "a trickling upload must surface DeadlineExceeded, got: {err}" + ); + // Stop trickling so the abandoned blocking task's reader errors and it can + // run its cleanup guard. + stop.store(true, Ordering::Relaxed); + + // The staging permit must have been released with the deadline: with a bound + // of 1, a blocked permit would hang this second upload forever. + zstd.update_zstd_oneshot( + digest, + DigestHasherFunc::Sha256, + Bytes::from(compressed.clone()), + ) + .await?; + // The abandoned blocking task cleans up once its reader errors, which happens + // asynchronously after the deadline fires. + while dir_entry_count(&temp) != 0 { + tokio::time::sleep(Duration::from_millis(20)).await; + } + Ok(()) +} + +// --------------------------------------------------------------------------- +// Commit against a `filesystem` backend (path-based commit, not descriptor) +// --------------------------------------------------------------------------- + +/// A `filesystem` backend drops the staged descriptor and commits by +/// `rename(2)` on the pathname, so the descriptor-pinning property that holds +/// for streaming backends does *not* apply to it. What must still hold: the +/// validated bytes land in the CAS, the staging directory is left empty, and the +/// blob reads back correctly through both the physical and identity views. +/// +/// `temp_path` deliberately sits next to the backend's `content_path` on one +/// filesystem: a cross-device staging directory would make the commit fail with +/// `EXDEV`. +#[nativelink_test] +async fn commit_to_filesystem_backend_round_trips() -> Result<(), Error> { + let data = compressible_data(96 * 1024); + let digest = digest_for(&data); + + let root = make_temp_path("zstd-fs-backend"); + let stage_path = format!("{root}/stage"); + let content_path = format!("{root}/content"); + let fs_temp_path = format!("{root}/fs-temp"); + for path in [&stage_path, &content_path, &fs_temp_path] { + std::fs::create_dir_all(path) + .map_err(|e| make_err!(Code::Internal, "Failed to create {path}: {e}"))?; + } + + let inner = Store::new( + FilesystemStore::::new(&FilesystemSpec { + content_path: content_path.clone(), + temp_path: fs_temp_path, + eviction_policy: None, + read_buffer_size: 0, + block_size: 0, + max_concurrent_writes: 0, + evict_page_cache: false, + }) + .await?, + ); + let mut spec = spec_for(stage_path.clone()); + spec.max_inline_commit_size = 1; // Force the staging + commit path. + let zstd = ZstdStore::new(&spec, inner.clone())?; + let store = Store::new(zstd.clone()); + + let compressed = zstd::bulk::compress(&data, 3).unwrap(); + zstd.update_zstd_oneshot( + digest, + DigestHasherFunc::Sha256, + Bytes::from(compressed.clone()), + ) + .await?; + + assert_eq!( + &inner.get_part_unchunked(digest, 0, None).await?[..], + &compressed[..], + "the filesystem backend must hold the validated zstd stream" + ); + assert_eq!( + &store.get_part_unchunked(digest, 0, None).await?[..], + &data[..], + "the committed blob must decode to the original content" + ); + assert_eq!( + collect_zstd(&zstd, digest).await?, + &compressed[..], + "passthrough reads must serve the stored stream byte-for-byte" + ); + assert_eq!( + dir_entry_count(&stage_path), + 0, + "a committed upload must leave the staging directory empty" + ); + Ok(()) +} diff --git a/nativelink-test/fuzz/Cargo.lock b/nativelink-test/fuzz/Cargo.lock index 95d52156d..69e81a584 100644 --- a/nativelink-test/fuzz/Cargo.lock +++ b/nativelink-test/fuzz/Cargo.lock @@ -17,9 +17,9 @@ dependencies = [ [[package]] name = "anyhow" -version = "1.0.102" +version = "1.0.104" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f202df86484c868dbad7eaa557ef785d5c66295e41b460ef922eca0723b842c" +checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470" [[package]] name = "arbitrary" @@ -58,7 +58,7 @@ checksum = "9035ad2d096bed7955a320ee7e2230574d28fd3c3a0f186cbea1ff3c7eed5dbb" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.118", ] [[package]] @@ -277,7 +277,7 @@ dependencies = [ "proc-macro2", "quote", "strsim", - "syn", + "syn 2.0.118", ] [[package]] @@ -288,7 +288,7 @@ checksum = "ac3984ec7bd6cfa798e62b4a642426a5be0e68f9401cfc2a01e3fa9ea2fcdb8d" dependencies = [ "darling_core", "quote", - "syn", + "syn 2.0.118", ] [[package]] @@ -305,7 +305,7 @@ checksum = "d65d7ce8132b7c0e54497a4d9a55a1c2a0912a0d786cf894472ba818fba45762" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.118", ] [[package]] @@ -316,7 +316,7 @@ checksum = "d08b3a0bcc0d079199cd476b2cae8435016ec11d1c0986c6901c5ac223041534" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.118", ] [[package]] @@ -338,7 +338,7 @@ dependencies = [ "proc-macro2", "quote", "rustc_version", - "syn", + "syn 2.0.118", "unicode-xid", ] @@ -361,7 +361,7 @@ checksum = "1ac70aa55017e108007fbaf5aa0f54b021c98f92ff8af59d42eda9da96e3dd4f" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.118", ] [[package]] @@ -463,7 +463,7 @@ checksum = "e835b70203e41293343137df5c0664546da5745f82ec9b84d40be8336958447b" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.118", ] [[package]] @@ -897,7 +897,7 @@ dependencies = [ "macro_magic_core", "macro_magic_macros", "quote", - "syn", + "syn 2.0.118", ] [[package]] @@ -911,7 +911,7 @@ dependencies = [ "macro_magic_core_macros", "proc-macro2", "quote", - "syn", + "syn 2.0.118", ] [[package]] @@ -922,7 +922,7 @@ checksum = "b02abfe41815b5bd98dbd4260173db2c116dda171dc0fe7838cb206333b83308" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.118", ] [[package]] @@ -933,7 +933,7 @@ checksum = "73ea28ee64b88876bf45277ed9a5817c1817df061a74f2b988971a12570e5869" dependencies = [ "macro_magic_core", "quote", - "syn", + "syn 2.0.118", ] [[package]] @@ -1033,12 +1033,12 @@ dependencies = [ "macro_magic", "proc-macro2", "quote", - "syn", + "syn 2.0.118", ] [[package]] name = "nativelink-config" -version = "1.6.1" +version = "1.6.3" dependencies = [ "byte-unit", "humantime", @@ -1053,7 +1053,7 @@ dependencies = [ [[package]] name = "nativelink-error" -version = "1.6.1" +version = "1.6.3" dependencies = [ "base64", "mongodb", @@ -1085,7 +1085,7 @@ dependencies = [ [[package]] name = "nativelink-metric" -version = "1.6.1" +version = "1.6.3" dependencies = [ "async-lock", "nativelink-metric-macro-derive", @@ -1096,16 +1096,16 @@ dependencies = [ [[package]] name = "nativelink-metric-macro-derive" -version = "1.6.1" +version = "1.6.3" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 3.0.1", ] [[package]] name = "nativelink-proto" -version = "1.6.1" +version = "1.6.3" dependencies = [ "derive_more", "prost", @@ -1209,7 +1209,7 @@ dependencies = [ "pest_meta", "proc-macro2", "quote", - "syn", + "syn 2.0.118", ] [[package]] @@ -1239,7 +1239,7 @@ checksum = "c96395f0a926bc13b1c17622aaddda1ecb55d49c8f1bf9777e4d877800a43f8b" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.118", ] [[package]] @@ -1301,7 +1301,7 @@ dependencies = [ "itertools", "proc-macro2", "quote", - "syn", + "syn 2.0.118", ] [[package]] @@ -1571,7 +1571,7 @@ checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.118", ] [[package]] @@ -1618,7 +1618,7 @@ dependencies = [ "darling", "proc-macro2", "quote", - "syn", + "syn 2.0.118", ] [[package]] @@ -1727,6 +1727,17 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "syn" +version = "3.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5edbec4ed188954a10c12c038215f8ce7606b2d5c973cd8dc43e8795065c5f2f" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + [[package]] name = "sync_wrapper" version = "1.0.2" @@ -1744,7 +1755,7 @@ checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.118", ] [[package]] @@ -1776,7 +1787,7 @@ checksum = "ebc4ee7f67670e9b64d05fa4253e753e016c6c95ff35b89b7941d6b856dec1d5" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.118", ] [[package]] @@ -1867,7 +1878,7 @@ checksum = "385a6cb71ab9ab790c5fe8d67f1645e6c450a7ce006a33de03daa956cf70a496" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.118", ] [[package]] @@ -2014,7 +2025,7 @@ checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.118", ] [[package]] @@ -2049,7 +2060,7 @@ checksum = "0e48cea23f68d1f78eb7bc092881b6bb88d3d6b5b7e6234f6f9c911da1ffb221" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.118", ] [[package]] @@ -2233,7 +2244,7 @@ dependencies = [ "bumpalo", "proc-macro2", "quote", - "syn", + "syn 2.0.118", "wasm-bindgen-shared", ] @@ -2408,7 +2419,7 @@ checksum = "de844c262c8848816172cef550288e7dc6c7b7814b4ee56b3e1553f275f1858e" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.118", "synstructure", ] @@ -2429,7 +2440,7 @@ checksum = "1ae7f38b72ec2a254e2b87ef277cf2cd4fb97cbebf944faa6f33354da0867930" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.118", ] [[package]] @@ -2449,7 +2460,7 @@ checksum = "11532158c46691caf0f2593ea8358fed6bbf68a0315e80aae9bd41fbade684a1" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.118", "synstructure", ] @@ -2489,7 +2500,7 @@ checksum = "625dc425cab0dca6dc3c3319506e6593dcb08a9f387ea3b284dbd52a92c40555" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.118", ] [[package]] diff --git a/nativelink-util/BUILD.bazel b/nativelink-util/BUILD.bazel index f9e0dcd0a..a7c5f499a 100644 --- a/nativelink-util/BUILD.bazel +++ b/nativelink-util/BUILD.bazel @@ -41,6 +41,7 @@ rust_library( "src/task.rs", "src/telemetry.rs", "src/tls_utils.rs", + "src/wire_compression.rs", ], proc_macro_deps = [ "@crates//:async-trait", @@ -90,6 +91,7 @@ rust_library( "@crates//:uuid", "@crates//:walkdir", "@crates//:wincode", + "@crates//:zstd", ], ) @@ -115,6 +117,7 @@ rust_test_suite( "tests/store_trait_test.rs", "tests/telemetry_test.rs", "tests/tls_utils_test.rs", + "tests/wire_compression_test.rs", ], compile_data = [ "tests/data/SekienAkashita.jpg", @@ -163,6 +166,7 @@ rust_test_suite( "@crates//:tracing", "@crates//:tracing-test", "@crates//:uuid", + "@crates//:zstd", ], ) diff --git a/nativelink-util/Cargo.toml b/nativelink-util/Cargo.toml index 09cdedf47..b665c5626 100644 --- a/nativelink-util/Cargo.toml +++ b/nativelink-util/Cargo.toml @@ -4,7 +4,7 @@ lints.workspace = true [package] edition = "2024" name = "nativelink-util" -version = "1.6.1" +version = "1.6.3" [dependencies] nativelink-config = { path = "../nativelink-config" } @@ -96,9 +96,10 @@ uuid = { version = "1.16.0", default-features = false, features = [ ] } walkdir = { version = "2.5.0", default-features = false } wincode = { version = "0.5.4", default-features = false, features = ["derive"] } +zstd = { version = "0.13.3", default-features = false } [dev-dependencies] -anyhow = { version = "1.0.100", default-features = false } +anyhow = { version = "1.0.103", default-features = false } nativelink-macro = { path = "../nativelink-macro" } opentelemetry-proto = { version = "0.32.0", default-features = false, features = [ "gen-tonic", diff --git a/nativelink-util/src/buf_channel.rs b/nativelink-util/src/buf_channel.rs index b685ca623..f198c74ac 100644 --- a/nativelink-util/src/buf_channel.rs +++ b/nativelink-util/src/buf_channel.rs @@ -429,6 +429,91 @@ impl DropCloserReadHalf { } } +/// A `std::io::Read` adapter over a [`DropCloserReadHalf`] that blocks the +/// current (blocking) thread while waiting for the next chunk. Only safe to use +/// from within `spawn_blocking!`. +#[derive(Debug)] +pub struct BufChannelReader { + rx: DropCloserReadHalf, + chunk: Bytes, + chunk_offset: usize, +} + +impl BufChannelReader { + #[must_use] + pub const fn new(rx: DropCloserReadHalf) -> Self { + Self { + rx, + chunk: Bytes::new(), + chunk_offset: 0, + } + } + + fn refill_chunk(&mut self) -> std::io::Result { + while self.chunk_offset == self.chunk.len() { + self.chunk = self.rx.blocking_recv().map_err(Error::to_std_err)?; + self.chunk_offset = 0; + if self.chunk.is_empty() { + return Ok(false); + } + } + Ok(true) + } +} + +impl std::io::Read for BufChannelReader { + fn read(&mut self, output: &mut [u8]) -> std::io::Result { + if output.is_empty() { + return Ok(0); + } + if !self.refill_chunk()? { + return Ok(0); + } + let chunk_remaining = &self.chunk[self.chunk_offset..]; + let bytes_to_copy = output.len().min(chunk_remaining.len()); + output[..bytes_to_copy].copy_from_slice(&chunk_remaining[..bytes_to_copy]); + self.chunk_offset += bytes_to_copy; + Ok(bytes_to_copy) + } +} + +/// A `std::io::Write` adapter over a [`DropCloserWriteHalf`] that blocks the +/// current (blocking) thread while forwarding chunks. Only safe to use from +/// within `spawn_blocking!`. EOF is intentionally NOT sent on drop; the caller +/// must call [`BufChannelWriter::send_eof`] explicitly once the write is +/// validated so a failed write never commits downstream. +#[derive(Debug)] +pub struct BufChannelWriter { + tx: DropCloserWriteHalf, +} + +impl BufChannelWriter { + #[must_use] + pub const fn new(tx: DropCloserWriteHalf) -> Self { + Self { tx } + } + + pub fn send_eof(&mut self) -> Result<(), Error> { + self.tx.send_eof() + } +} + +impl std::io::Write for BufChannelWriter { + fn write(&mut self, buf: &[u8]) -> std::io::Result { + if buf.is_empty() { + return Ok(0); + } + self.tx + .blocking_send(Bytes::copy_from_slice(buf)) + .map_err(Error::to_std_err)?; + Ok(buf.len()) + } + + fn flush(&mut self) -> std::io::Result<()> { + Ok(()) + } +} + impl Stream for DropCloserReadHalf { type Item = Result; diff --git a/nativelink-util/src/evicting_map.rs b/nativelink-util/src/evicting_map.rs index bd73b86cc..3339a6597 100644 --- a/nativelink-util/src/evicting_map.rs +++ b/nativelink-util/src/evicting_map.rs @@ -288,7 +288,7 @@ where anchor_time, max_bytes: config.max_bytes as u64, evict_bytes: config.evict_bytes as u64, - max_seconds: config.max_seconds as i32, + max_seconds: config.max_seconds.try_into().unwrap_or(i32::MAX), max_count: config.max_count, } } diff --git a/nativelink-util/src/fs.rs b/nativelink-util/src/fs.rs index 47c5ccf3f..b271d5932 100644 --- a/nativelink-util/src/fs.rs +++ b/nativelink-util/src/fs.rs @@ -44,6 +44,23 @@ pub struct FileSlot { } impl FileSlot { + /// Wrap an already-open [`std::fs::File`] that is accounted for by `permit` + /// (a permit from [`OPEN_FILE_SEMAPHORE`], e.g. via [`get_permit`]). + /// + /// This is the escape hatch for callers that open/create a descriptor inside + /// their own `spawn_blocking!` closure (for example, to write synchronously + /// with the [`std::io`] API) and later need to hand the *same* descriptor to + /// an async consumer such as [`crate::store_trait::StoreLike::update_with_whole_file`]. + /// The caller is responsible for having acquired `permit` before opening the + /// file so the descriptor stays accounted for by the global semaphore. + #[must_use] + pub fn from_std(permit: SemaphorePermit<'static>, file: std::fs::File) -> Self { + Self { + _permit: permit, + inner: tokio::fs::File::from_std(file), + } + } + /// Advise the kernel to drop page cache for this file's contents. /// Only available on Linux; #[cfg(target_os = "linux")] diff --git a/nativelink-util/src/lib.rs b/nativelink-util/src/lib.rs index 932c21bb0..d082f3c3a 100644 --- a/nativelink-util/src/lib.rs +++ b/nativelink-util/src/lib.rs @@ -39,6 +39,7 @@ pub mod store_trait; pub mod task; pub mod telemetry; pub mod tls_utils; +pub mod wire_compression; // Re-export tracing mostly for use in macros. pub use tracing as __tracing; diff --git a/nativelink-util/src/metrics.rs b/nativelink-util/src/metrics.rs index c04a75cc7..99838e5fd 100644 --- a/nativelink-util/src/metrics.rs +++ b/nativelink-util/src/metrics.rs @@ -15,10 +15,11 @@ // limitations under the License. use std::sync::LazyLock; +use std::time::SystemTime; use opentelemetry::{InstrumentationScope, KeyValue, Value, global, metrics}; -use crate::action_messages::ActionStage; +use crate::action_messages::{ActionResult, ActionStage}; // Metric attribute keys for cache operations. pub const CACHE_TYPE: &str = "cache.type"; @@ -566,39 +567,6 @@ pub static EXECUTION_METRICS: LazyLock = LazyLock::new(|| { ]) .build(), - execution_cpu_time: meter - .f64_histogram("execution.cpu.time") - .with_description("CPU time consumed by action execution in seconds") - .with_unit("s") - .with_boundaries(vec![ - 0.01, // 10ms - 0.1, // 100ms - 1.0, // 1s - 10.0, // 10s - 60.0, // 1 minute - 300.0, // 5 minutes - 600.0, // 10 minutes - 1800.0, // 30 minutes - 3600.0, // 1 hour - ]) - .build(), - - execution_memory_usage: meter - .u64_histogram("execution.memory.usage") - .with_description("Peak memory usage during execution in bytes") - .with_unit("By") - .with_boundaries(vec![ - 1_048_576.0, // 1MB - 10_485_760.0, // 10MB - 104_857_600.0, // 100MB - 524_288_000.0, // 500MB - 1_073_741_824.0, // 1GB - 5_368_709_120.0, // 5GB - 10_737_418_240.0, // 10GB - 53_687_091_200.0, // 50GB - ]) - .build(), - execution_retry_count: meter .u64_counter("execution.retry.count") .with_description("Number of execution retries") @@ -624,10 +592,6 @@ pub struct ExecutionMetrics { pub execution_stage_transitions: metrics::Counter, /// Histogram of output sizes in bytes pub execution_output_size: metrics::Histogram, - /// Histogram of CPU time in seconds - pub execution_cpu_time: metrics::Histogram, - /// Histogram of peak memory usage in bytes - pub execution_memory_usage: metrics::Histogram, /// Counter for execution retries pub execution_retry_count: metrics::Counter, } @@ -651,3 +615,78 @@ pub fn make_execution_attributes( attrs } + +/// Records the histogram metrics derivable from a completed action's result. +pub fn record_completed_execution_metrics( + action_result: &ActionResult, + instance_name: &str, + worker_id: Option<&str>, + priority: Option, +) { + let m = &*EXECUTION_METRICS; + let md = &action_result.execution_metadata; + let base = make_execution_attributes(instance_name, worker_id, priority); + + let record_secs = + |hist: &metrics::Histogram, start: SystemTime, end: SystemTime, attrs: &[KeyValue]| { + if start > SystemTime::UNIX_EPOCH + && let Ok(d) = end.duration_since(start) + { + hist.record(d.as_secs_f64(), attrs); + } + }; + + // Queue wait (queued -> worker picked it up) and end-to-end duration. + record_secs( + &m.execution_queue_time, + md.queued_timestamp, + md.worker_start_timestamp, + &base, + ); + record_secs( + &m.execution_total_duration, + md.queued_timestamp, + md.worker_completed_timestamp, + &base, + ); + + // Per-phase stage durations, labeled by phase on the stage attribute. + for (phase, start, end) in [ + ( + "input_fetch", + md.input_fetch_start_timestamp, + md.input_fetch_completed_timestamp, + ), + ( + "execution", + md.execution_start_timestamp, + md.execution_completed_timestamp, + ), + ( + "output_upload", + md.output_upload_start_timestamp, + md.output_upload_completed_timestamp, + ), + ] { + let mut attrs = base.clone(); + attrs.push(KeyValue::new(EXECUTION_STAGE, phase)); + record_secs(&m.execution_stage_duration, start, end, &attrs); + } + + // Total bytes produced: output files plus stdout/stderr. + m.execution_output_size + .record(execution_output_bytes(action_result), &base); +} + +/// Total output bytes produced by an action: the output file digests plus the +/// stdout and stderr digests. +#[must_use] +pub fn execution_output_bytes(action_result: &ActionResult) -> u64 { + action_result + .output_files + .iter() + .map(|f| f.digest.size_bytes()) + .sum::() + + action_result.stdout_digest.size_bytes() + + action_result.stderr_digest.size_bytes() +} diff --git a/nativelink-util/src/proto_stream_utils.rs b/nativelink-util/src/proto_stream_utils.rs index 08362005d..39878c48d 100644 --- a/nativelink-util/src/proto_stream_utils.rs +++ b/nativelink-util/src/proto_stream_utils.rs @@ -227,6 +227,12 @@ where resume_queue: [Option; 2], // An optimisation to avoid having to manage resume_queue when it's empty. is_resumed: bool, + // When false, a partially-consumed stream never reports `can_resume()`: + // uploads whose server-side protocol cannot accept a replay from a + // nonzero offset (REAPI compressed-blobs writes) must fail fast instead + // of burning retries on guaranteed-rejected resumes. A stream that has + // not yet been consumed can always be retried from the start. + resumable: bool, } impl WriteState @@ -242,9 +248,23 @@ where cached_messages: [None, None], resume_queue: [None, None], is_resumed: false, + resumable: true, } } + /// Marks this write as non-resumable: once the stream has been partially + /// consumed, `can_resume()` reports false so the caller fails fast with + /// the original error instead of replaying messages the server-side + /// protocol is guaranteed to reject. Retrying an unconsumed stream from + /// the start remains allowed. + pub const fn set_non_resumable(&mut self) { + self.resumable = false; + } + + pub(crate) const fn is_resumable(&self) -> bool { + self.resumable + } + fn push_message(&mut self, message: WriteRequest) { self.cached_messages.swap(0, 1); self.cached_messages[0] = Some(message); @@ -267,7 +287,8 @@ where pub const fn can_resume(&self) -> bool { self.read_stream_error.is_none() - && (self.cached_messages[0].is_some() || self.read_stream.is_first_msg()) + && ((self.resumable && self.cached_messages[0].is_some()) + || self.read_stream.is_first_msg()) } pub fn resume(&mut self) { @@ -345,8 +366,11 @@ where } } // Cache the last request in case there is an error to allow - // the upload to be resumed. - local_state.push_message(message.clone()); + // the upload to be resumed. Non-resumable writes skip the + // clone: cached messages would never be replayed. + if local_state.is_resumable() { + local_state.push_message(message.clone()); + } Some(message) } Some(Err(err)) => { diff --git a/nativelink-util/src/store_trait.rs b/nativelink-util/src/store_trait.rs index 942d3a15c..1524a8891 100644 --- a/nativelink-util/src/store_trait.rs +++ b/nativelink-util/src/store_trait.rs @@ -353,6 +353,63 @@ impl Display for StoreKey<'_> { } } +/// A wire representation understood by a [`WireCompressionStore`]. +/// +/// This is intentionally separate from the REAPI protobuf enum: stores own +/// their physical representation, while protocol adapters translate their +/// negotiated compressor to this util-level capability. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub enum WireCompressor { + /// A standard zstd frame. + Zstd, +} + +/// Optional immediate-store capability for serving and accepting a compressed +/// wire representation without a decode/re-encode round trip. +/// +/// A `StoreDriver` exposes this capability only when it is itself the store +/// that owns the physical representation. Wrappers must not forward it: their +/// own transformation may make the wrapped bytes unsuitable for wire +/// passthrough. +#[async_trait] +pub trait WireCompressionStore: Send + Sync + 'static { + /// Stores a client-supplied compressed stream after validating it against + /// the declared digest. Returns the number of compressed wire bytes stored. + async fn update_compressed( + self: Arc, + digest: DigestInfo, + digest_function: DigestHasherFunc, + compressor: WireCompressor, + reader: DropCloserReadHalf, + ) -> Result; + + /// Streams the stored compressed representation for a blob. + async fn get_compressed( + self: Arc, + digest: DigestInfo, + compressor: WireCompressor, + writer: DropCloserWriteHalf, + ) -> Result<(), Error>; + + /// Stores a complete client-supplied compressed blob after validation. + async fn update_compressed_oneshot( + self: Arc, + digest: DigestInfo, + digest_function: DigestHasherFunc, + compressor: WireCompressor, + data: Bytes, + ) -> Result<(), Error>; + + /// Returns either a stored compressed representation accepted by the + /// client, or the decoded identity representation. The optional compressor + /// describes `data`; `None` means identity. + async fn get_for_batch( + self: Arc, + digest: DigestInfo, + acceptable_compressors: &[WireCompressor], + ) -> Result<(Bytes, Option), Error>; +} + #[derive(Clone, MetricsComponent)] #[repr(transparent)] pub struct Store { @@ -394,6 +451,16 @@ impl Store { self.inner.inner_store(maybe_digest).as_any().downcast_ref() } + /// Returns the wire-compression capability of the immediate store driver. + /// + /// This deliberately does not follow [`Self::inner_store`]. A wrapper + /// outside a representation-changing store must disable passthrough unless + /// it explicitly exposes its own valid wire representation. + #[inline] + pub fn wire_compression_store(&self) -> Option> { + self.inner.clone().wire_compression_store() + } + /// Register health checks used to monitor the store. #[inline] pub fn register_health(&self, registry: &mut HealthRegistryBuilder) { @@ -401,10 +468,7 @@ impl Store { } #[inline] - pub fn register_remove_callback( - &self, - callback: Arc, - ) -> Result<(), Error> { + pub fn register_remove_callback(&self, callback: RemoveCallback) -> Result<(), Error> { self.inner.clone().register_remove_callback(callback) } } @@ -617,6 +681,8 @@ pub trait StoreLike: Send + Sync + Sized + Unpin + 'static { } } +pub type RemoveCallback = Arc; + #[async_trait] pub trait StoreDriver: Sync + Send + Unpin + MetricsComponent + HealthStatusIndicator + 'static @@ -625,6 +691,12 @@ pub trait StoreDriver: // for ref stores async fn post_init(self: Arc) -> Result<(), Error>; + /// Returns an optional capability for the driver's immediate compressed + /// wire representation. The default intentionally disables passthrough. + fn wire_compression_store(self: Arc) -> Option> { + None + } + /// See: [`StoreLike::has`] for details. #[inline] async fn has(self: Pin<&Self>, key: StoreKey<'_>) -> Result, Error> { @@ -865,10 +937,7 @@ pub trait StoreDriver: // Register health checks used to monitor the store. fn register_health(self: Arc, _registry: &mut HealthRegistryBuilder) {} - fn register_remove_callback( - self: Arc, - callback: Arc, - ) -> Result<(), Error>; + fn register_remove_callback(self: Arc, callback: RemoveCallback) -> Result<(), Error>; } // Callback to be called when a store deletes an item. This is used so diff --git a/nativelink-util/src/telemetry.rs b/nativelink-util/src/telemetry.rs index 527d8473d..d0de0d0e5 100644 --- a/nativelink-util/src/telemetry.rs +++ b/nativelink-util/src/telemetry.rs @@ -118,7 +118,7 @@ pub async fn init_tracing() -> Result<(), nativelink_error::Error> { // We currently use a UUIDv4 for "service.instance.id" as per: // https://opentelemetry.io/docs/specs/semconv/attributes-registry/service/ - // This might change as we get a better understanding of its usecases in the + // This might change as we get a better understanding of its use cases in the // context of broader observability infrastructure. let resource = Resource::builder() .with_service_name(NATIVELINK_SERVICE_NAME) @@ -246,8 +246,7 @@ pub async fn maybe_load_balanced_channel() -> Option { } } /// This is the header that bazel sends when using the `--remote_header` flag. -/// TODO(palfrey): There are various other headers that bazel supports. -/// Optimize their usage. +/// TODO(palfrey): Bazel supports other headers, and we should optimize their usage. const BAZEL_REQUESTMETADATA_HEADER: &str = "build.bazel.remote.execution.v2.requestmetadata-bin"; use opentelemetry::baggage::BaggageExt; @@ -297,7 +296,7 @@ where fn call(&mut self, req: hyper::http::Request) -> Self::Future { // We must take the current `inner` and not the clone. - // See: https://docs.rs/tower/latest/tower/trait.Service.html#be-careful-when-cloning-inner-services + // See: let clone = self.inner.clone(); let mut inner = core::mem::replace(&mut self.inner, clone); diff --git a/nativelink-util/src/wire_compression.rs b/nativelink-util/src/wire_compression.rs new file mode 100644 index 000000000..d301cf65d --- /dev/null +++ b/nativelink-util/src/wire_compression.rs @@ -0,0 +1,364 @@ +// Copyright 2026 The NativeLink Authors. All rights reserved. +// +// Licensed under the Functional Source License, Version 1.1, Apache 2.0 Future License (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// See LICENSE file for details +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Zstd wire-compression codecs for REAPI compressed-blobs. +//! +//! Shared by the server-side accept/serve paths (nativelink-service) and the +//! client-side `GrpcStore` transfers (nativelink-store). This is orthogonal +//! to at-rest compression (`CompressionStore` with LZ4). + +use std::io::Read; + +use bytes::Bytes; +use nativelink_error::{Code, Error, ResultExt, make_err, make_input_err}; +use nativelink_proto::build::bazel::remote::execution::v2::compressor; + +use crate::buf_channel::{DropCloserReadHalf, DropCloserWriteHalf}; +use crate::common::DigestInfo; +use crate::digest_hasher::{DigestHasher, DigestHasherFunc}; +use crate::store_trait::WireCompressor; + +/// Zstd compression level for wire compression. +/// Level 0 in the zstd crate means "use default" (currently 3). +/// We use an explicit level for clarity. +pub const ZSTD_COMPRESSION_LEVEL: i32 = 3; + +/// Upper bound on the buffer `decompress` reserves up front for a zstd blob. +/// `expected_size` comes from the client's claimed digest, so it must never be +/// used as an allocation hint directly: a small payload claiming a huge size +/// would otherwise force a large pre-emptive allocation before the real +/// decompressed length is ever known. We reserve `min(expected_size, this)` +/// so common payloads never reallocate while a hostile claim allocates at most +/// this. Sized comfortably above any honest `BatchUpdateBlobs` payload (the +/// only caller of this bulk path; large blobs stream through `ByteStream`). +const ZSTD_DECOMPRESS_PREALLOC_CAP: usize = 1024 * 1024; + +/// Compress data using the specified wire compressor. +/// +/// `data` is the raw (uncompressed) bytes from the store. +/// Returns the compressed bytes suitable for sending on the wire. +pub fn compress(data: Bytes, compressor_value: compressor::Value) -> Result { + match compressor_value { + compressor::Value::Identity => Ok(data), + compressor::Value::Zstd => { + let compressed = zstd::bulk::compress(&data, ZSTD_COMPRESSION_LEVEL) + .map_err(|e| make_err!(Code::Internal, "Zstd compression failed: {}", e))?; + Ok(Bytes::from(compressed)) + } + _ => Err(make_input_err!( + "Unsupported wire compressor for compression: {:?}", + compressor_value + )), + } +} + +/// Decompress data using the specified wire compressor. +/// +/// `data` is the compressed bytes received from the wire. +/// `expected_size` is the uncompressed size (from the client's digest). It is +/// the hard cap on the decompressed output, but never a direct allocation +/// hint: the buffer grows with the real decoded bytes so a small payload +/// claiming a huge size cannot force a large up-front allocation. +/// Returns the decompressed bytes suitable for storing. +pub fn decompress( + data: &[u8], + compressor_value: compressor::Value, + expected_size: usize, +) -> Result { + match compressor_value { + compressor::Value::Identity => { + if data.len() != expected_size { + return Err(make_err!( + Code::InvalidArgument, + "Identity data size {} does not match expected size {}", + data.len(), + expected_size + )); + } + Ok(Bytes::copy_from_slice(data)) + } + compressor::Value::Zstd => { + // Decode incrementally so `expected_size` (which is attacker + // controlled — it is the client's claimed digest size) can bound + // the output without being trusted as an allocation size. We + // reserve only `min(expected_size, ZSTD_DECOMPRESS_PREALLOC_CAP)`, + // then `take(expected_size + 1)` hard-caps the decoder so a + // decompression bomb is rejected as soon as it overshoots. This + // mirrors the real-byte-count validation the identity arm and the + // streaming upload path already perform. + let decoder = zstd::stream::read::Decoder::new(data) + .map_err(|e| make_err!(Code::InvalidArgument, "Zstd decompression failed: {e}"))?; + let mut output = Vec::with_capacity(expected_size.min(ZSTD_DECOMPRESS_PREALLOC_CAP)); + // `+ 1` lets an oversized stream produce one byte past the cap so + // the size check below rejects it rather than silently truncating. + let cap = u64::try_from(expected_size) + .err_tip(|| "expected_size did not fit in u64")? + .saturating_add(1); + decoder + .take(cap) + .read_to_end(&mut output) + .map_err(|e| make_err!(Code::InvalidArgument, "Zstd decompression failed: {e}"))?; + if output.len() != expected_size { + return Err(make_err!( + Code::InvalidArgument, + "Decompressed size {} does not match expected size {}", + output.len(), + expected_size + )); + } + Ok(Bytes::from(output)) + } + _ => Err(make_input_err!( + "Unsupported wire compressor for decompression: {:?}", + compressor_value + )), + } +} + +/// Decode a client's zstd wire stream into raw bytes on `tx`, asynchronously. +/// +/// Like [`stream_encode_compressed_download`], this must not occupy a tokio +/// blocking-pool thread for the stream's lifetime: the input arrives at the +/// client's upload pace and `tx` drains at the store's write pace, so a +/// blocking implementation parks a pool thread on whichever side is slower +/// for as long as the upload lasts. The zstd frame is consumed incrementally +/// with the raw streaming API instead; per-chunk decode cost is bounded by +/// the channel chunk size, so it runs inline on the async runtime with +/// channel-native backpressure on both sides. +/// +/// Validation semantics match the REAPI compressed-blobs contract: the +/// decoded byte count may never exceed the digest size (checked per chunk so +/// a decompression bomb is rejected as soon as it overshoots), the final +/// count must equal it exactly, and the decoded bytes must hash to `digest`. +pub async fn stream_decode_compressed_upload( + mut compressed_rx: DropCloserReadHalf, + wire_compressor: compressor::Value, + digest: DigestInfo, + digest_function: DigestHasherFunc, + mut tx: DropCloserWriteHalf, +) -> Result<(), Error> { + use zstd::stream::raw::{Decoder, InBuffer, Operation, OutBuffer}; + + if wire_compressor != compressor::Value::Zstd { + return Err(make_input_err!( + "Streaming upload decompression only supports zstd, got {:?}", + wire_compressor + )); + } + + let expected_size = digest.size_bytes(); + let mut hasher = digest_function.hasher(); + let mut decoded_size = 0u64; + let mut decoder = Decoder::new() + .map_err(|e| make_err!(Code::InvalidArgument, "Zstd decompression failed: {}", e))?; + // `DCtx::out_size()` guarantees a full decompressed block always fits, so + // the decoder never stalls for lack of output space within one `run`. + let mut out_buf = vec![0u8; zstd::zstd_safe::DCtx::out_size()]; + // Last input-size hint from the decoder: nonzero at input EOF means the + // stream ended in the middle of a frame and must be rejected. + let mut frame_input_hint = 0usize; + loop { + let chunk = compressed_rx + .recv() + .await + .err_tip(|| "Failed to receive compressed data in stream_decode_compressed_upload")?; + if chunk.is_empty() { + break; // EOF. + } + let mut in_buffer = InBuffer::around(&chunk); + loop { + let mut out_buffer = OutBuffer::around(out_buf.as_mut_slice()); + frame_input_hint = decoder.run(&mut in_buffer, &mut out_buffer).map_err(|e| { + make_err!(Code::InvalidArgument, "Zstd decompression failed: {}", e) + })?; + let produced = Bytes::copy_from_slice(out_buffer.as_slice()); + // A completely full output buffer means the decoder may still + // have buffered output to flush, even with no input left. + let output_was_full = produced.len() == out_buf.len(); + if !produced.is_empty() { + let produced_u64 = u64::try_from(produced.len()) + .err_tip(|| "Decoded chunk size was not convertible to u64")?; + decoded_size = decoded_size.checked_add(produced_u64).ok_or_else(|| { + make_err!( + Code::InvalidArgument, + "Decoded compressed upload size overflow" + ) + })?; + if decoded_size > expected_size { + return Err(make_err!( + Code::InvalidArgument, + "Decoded compressed upload size {} bytes exceeds digest size {} bytes", + decoded_size, + expected_size + )); + } + hasher.update(&produced); + tx.send(produced).await?; + } + // `hint == 0` means the frame is completely decoded AND fully + // flushed. It must terminate the loop even when the output + // buffer was filled exactly: polling the decoder again after + // frame end would return the input-size hint for a NEW frame + // header, and the post-EOF `frame_input_hint != 0` check would + // then misreport a fully-decoded stream as truncated. This is + // deterministic for blobs whose decompressed size is an exact + // multiple of the decoder output buffer size. + if in_buffer.pos() == in_buffer.src.len() && (frame_input_hint == 0 || !output_was_full) + { + break; + } + } + } + if frame_input_hint != 0 { + return Err(make_err!( + Code::InvalidArgument, + "Compressed upload stream ended in the middle of a zstd frame" + )); + } + + if decoded_size != expected_size { + return Err(make_err!( + Code::InvalidArgument, + "Decompressed size {} does not match expected size {}", + decoded_size, + expected_size + )); + } + let actual_digest = hasher.finalize_digest(); + if actual_digest != digest { + return Err(make_err!( + Code::InvalidArgument, + "Decompressed digest {} does not match expected digest {}", + actual_digest, + digest + )); + } + + tx.send_eof() + .err_tip(|| "Failed to send decompressed upload EOF")?; + Ok(()) +} + +/// Encode a raw byte stream into a single zstd frame on `tx`, asynchronously. +/// +/// This runs entirely on the async runtime and must never occupy a tokio +/// blocking-pool thread for the stream's lifetime: `tx` drains at the gRPC +/// client's pace, so a blocking implementation (blocking reads from `raw_rx` +/// plus `blocking_send` into `tx`) parks one pool thread per concurrent +/// compressed download until the client finishes. Enough concurrent downloads +/// with slow consumers then exhaust the blocking pool and starve every other +/// `spawn_blocking` user (filesystem store I/O, upload decode, credential +/// resolution). The zstd frame is instead produced incrementally with the raw +/// streaming API: the CPU cost per iteration is bounded by the channel chunk +/// size (small — micro/milliseconds), so it is acceptable inline on a worker +/// thread, and `tx.send(...).await` gives backpressure without a parked +/// thread. +/// +/// The output is one well-formed zstd frame, identical in wire format to what +/// `zstd::stream::read::Encoder` produces (both drive `ZSTD_compressStream` +/// on a fresh `CCtx`). +pub async fn stream_encode_compressed_download( + mut raw_rx: DropCloserReadHalf, + wire_compressor: compressor::Value, + compression_level: i32, + tx: DropCloserWriteHalf, +) -> Result<(), Error> { + stream_encode_compressed_download_from_reader( + &mut raw_rx, + wire_compressor, + compression_level, + tx, + ) + .await +} + +/// Encode a raw byte stream into a single zstd frame, borrowing the input +/// reader so a caller can continue draining it if the downstream consumer +/// finishes before the encoder does. +pub async fn stream_encode_compressed_download_from_reader( + raw_rx: &mut DropCloserReadHalf, + wire_compressor: compressor::Value, + compression_level: i32, + mut tx: DropCloserWriteHalf, +) -> Result<(), Error> { + use zstd::stream::raw::{Encoder, InBuffer, Operation, OutBuffer}; + + if wire_compressor != compressor::Value::Zstd { + return Err(make_input_err!( + "Streaming download compression only supports zstd, got {:?}", + wire_compressor + )); + } + + let mut encoder = Encoder::new(compression_level) + .map_err(|e| make_err!(Code::Internal, "Zstd compression failed: {}", e))?; + // `CCtx::out_size()` guarantees a full compressed block always fits, so + // the encoder never stalls for lack of output space within one `run`. + let mut out_buf = vec![0u8; zstd::zstd_safe::CCtx::out_size()]; + loop { + let chunk = raw_rx + .recv() + .await + .err_tip(|| "Failed to receive raw data in stream_encode_compressed_download")?; + if chunk.is_empty() { + break; // EOF. + } + let mut in_buffer = InBuffer::around(&chunk); + while in_buffer.pos() < in_buffer.src.len() { + let mut out_buffer = OutBuffer::around(out_buf.as_mut_slice()); + encoder + .run(&mut in_buffer, &mut out_buffer) + .map_err(|e| make_err!(Code::Internal, "Zstd compression failed: {}", e))?; + let produced = out_buffer.as_slice(); + if !produced.is_empty() { + tx.send(Bytes::copy_from_slice(produced)).await?; + } + } + } + + // Finish the frame: flush any internally buffered compressed data plus + // the frame epilogue. `finish` reports the bytes still pending, so loop + // until it reports none. + loop { + let mut out_buffer = OutBuffer::around(out_buf.as_mut_slice()); + let remaining = encoder + .finish(&mut out_buffer, true) + .map_err(|e| make_err!(Code::Internal, "Zstd compression failed: {}", e))?; + let produced = out_buffer.as_slice(); + if !produced.is_empty() { + tx.send(Bytes::copy_from_slice(produced)).await?; + } + if remaining == 0 { + break; + } + } + + tx.send_eof() + .err_tip(|| "Failed to send compressed download EOF")?; + Ok(()) +} + +/// Translate a negotiated REAPI compressor into the store-level wire +/// representation, or `None` when no store can serve that compressor's bytes +/// directly. Keeps the passthrough fast paths from assuming a compressor the +/// client did not actually negotiate. +#[must_use] +pub const fn wire_compressor_capability( + compressor_value: compressor::Value, +) -> Option { + match compressor_value { + compressor::Value::Zstd => Some(WireCompressor::Zstd), + _ => None, + } +} diff --git a/nativelink-util/tests/action_messages_test.rs b/nativelink-util/tests/action_messages_test.rs index f295ca08d..963414a34 100644 --- a/nativelink-util/tests/action_messages_test.rs +++ b/nativelink-util/tests/action_messages_test.rs @@ -66,7 +66,7 @@ fn make_missing_blob_error(digest: &DigestInfo) -> Error { ) .with_context(ErrorContext::MissingDigest { hash: digest.packed_hash().to_string(), - size: digest.size_bytes() as i64, + size: digest.size_bytes().try_into().unwrap_or(i64::MAX), }) } diff --git a/nativelink-util/tests/metrics_test.rs b/nativelink-util/tests/metrics_test.rs index d4e83f1a6..460f04421 100644 --- a/nativelink-util/tests/metrics_test.rs +++ b/nativelink-util/tests/metrics_test.rs @@ -12,10 +12,13 @@ // See the License for the specific language governing permissions and // limitations under the License. -use nativelink_util::action_messages::{ActionResult, ActionStage}; +use nativelink_util::action_messages::{ + ActionResult, ActionStage, ExecutionMetadata, FileInfo, NameOrPath, +}; +use nativelink_util::common::DigestInfo; use nativelink_util::metrics::{ CACHE_METRICS, CacheMetricAttrs, EXECUTION_METRICS, ExecutionMetricAttrs, ExecutionStage, - make_execution_attributes, + execution_output_bytes, make_execution_attributes, record_completed_execution_metrics, }; use opentelemetry::KeyValue; @@ -195,3 +198,55 @@ fn test_action_stage_conversion_avoids_clone() { "Reference conversion took too long: {elapsed:?}" ); } + +#[test] +fn execution_output_bytes_sums_files_and_streams() { + let hash = "0123456789abcdef000000000000000000010000000000000123456789abcdef"; + let result = ActionResult { + output_files: vec![ + FileInfo { + name_or_path: NameOrPath::Path("out1".to_string()), + digest: DigestInfo::try_new(hash, 1000).unwrap(), + is_executable: false, + }, + FileInfo { + name_or_path: NameOrPath::Path("out2".to_string()), + digest: DigestInfo::try_new(hash, 234).unwrap(), + is_executable: false, + }, + ], + stdout_digest: DigestInfo::try_new(hash, 10).unwrap(), + stderr_digest: DigestInfo::try_new(hash, 20).unwrap(), + ..ActionResult::default() + }; + assert_eq!(execution_output_bytes(&result), 1000 + 234 + 10 + 20); +} + +#[test] +fn record_completed_execution_metrics_guards_unpopulated_metadata() { + use core::time::Duration; + use std::time::SystemTime; + + let queued = SystemTime::UNIX_EPOCH + Duration::from_secs(1_000_000); + let at = |n| queued + Duration::from_secs(n); + let populated = ActionResult { + execution_metadata: ExecutionMetadata { + queued_timestamp: queued, + worker_start_timestamp: at(2), + worker_completed_timestamp: at(12), + input_fetch_start_timestamp: at(2), + input_fetch_completed_timestamp: at(4), + execution_start_timestamp: at(4), + execution_completed_timestamp: at(11), + output_upload_start_timestamp: at(11), + output_upload_completed_timestamp: at(12), + ..ExecutionMetadata::default() + }, + ..ActionResult::default() + }; + // Should record without panicking. + record_completed_execution_metrics(&populated, "instance", Some("worker-1"), Some(3)); + + // Default (UNIX_EPOCH) metadata must skip the duration records cleanly. + record_completed_execution_metrics(&ActionResult::default(), "instance", None, None); +} diff --git a/nativelink-util/tests/origin_event_test.rs b/nativelink-util/tests/origin_event_test.rs index 5404ad278..9f72e7d05 100644 --- a/nativelink-util/tests/origin_event_test.rs +++ b/nativelink-util/tests/origin_event_test.rs @@ -29,9 +29,7 @@ use nativelink_util::default_health_status_indicator; use nativelink_util::health_utils::HealthStatusIndicator; use nativelink_util::origin_event::get_id_for_event; use nativelink_util::origin_event_publisher::OriginEventPublisher; -use nativelink_util::store_trait::{ - RemoveItemCallback, Store, StoreDriver, StoreKey, UploadSizeInfo, -}; +use nativelink_util::store_trait::{RemoveCallback, Store, StoreDriver, StoreKey, UploadSizeInfo}; use tokio::sync::{broadcast, mpsc}; use tokio::time::sleep; use tonic::async_trait; @@ -100,10 +98,7 @@ impl StoreDriver for FlakyStore { self } - fn register_remove_callback( - self: Arc, - _callback: Arc, - ) -> Result<(), Error> { + fn register_remove_callback(self: Arc, _callback: RemoveCallback) -> Result<(), Error> { todo!(); } } diff --git a/nativelink-util/tests/store_trait_test.rs b/nativelink-util/tests/store_trait_test.rs index c09b03838..83dba428e 100644 --- a/nativelink-util/tests/store_trait_test.rs +++ b/nativelink-util/tests/store_trait_test.rs @@ -1,14 +1,18 @@ use core::pin::Pin; use std::sync::Arc; -use nativelink_error::Error; +use bytes::Bytes; +use nativelink_error::{Code, Error, make_err}; use nativelink_macro::nativelink_test; use nativelink_metric::MetricsComponent; use nativelink_util::buf_channel::{DropCloserReadHalf, DropCloserWriteHalf}; +use nativelink_util::common::DigestInfo; use nativelink_util::default_health_status_indicator; +use nativelink_util::digest_hasher::DigestHasherFunc; use nativelink_util::health_utils::HealthStatusIndicator; use nativelink_util::store_trait::{ - RemoveItemCallback, Store, StoreDriver, StoreKey, StoreLike, UploadSizeInfo, + RemoveCallback, Store, StoreDriver, StoreKey, StoreLike, UploadSizeInfo, WireCompressionStore, + WireCompressor, }; use tonic::async_trait; @@ -22,6 +26,10 @@ impl StoreDriver for FakeStore { Ok(()) } + fn wire_compression_store(self: Arc) -> Option> { + Some(self) + } + async fn has_with_results( self: Pin<&Self>, _keys: &[StoreKey<'_>], @@ -61,15 +69,130 @@ impl StoreDriver for FakeStore { self } - fn register_remove_callback( + fn register_remove_callback(self: Arc, _callback: RemoveCallback) -> Result<(), Error> { + todo!(); + } +} + +default_health_status_indicator!(FakeStore); + +#[async_trait] +impl WireCompressionStore for FakeStore { + async fn update_compressed( + self: Arc, + _digest: DigestInfo, + _digest_function: DigestHasherFunc, + _compressor: WireCompressor, + _reader: DropCloserReadHalf, + ) -> Result { + Err(make_err!(Code::Unimplemented, "fake wire store")) + } + + async fn get_compressed( + self: Arc, + _digest: DigestInfo, + _compressor: WireCompressor, + _writer: DropCloserWriteHalf, + ) -> Result<(), Error> { + Err(make_err!(Code::Unimplemented, "fake wire store")) + } + + async fn update_compressed_oneshot( self: Arc, - _callback: Arc, + _digest: DigestInfo, + _digest_function: DigestHasherFunc, + _compressor: WireCompressor, + _data: Bytes, + ) -> Result<(), Error> { + Err(make_err!(Code::Unimplemented, "fake wire store")) + } + + async fn get_for_batch( + self: Arc, + _digest: DigestInfo, + _acceptable_compressors: &[WireCompressor], + ) -> Result<(Bytes, Option), Error> { + Err(make_err!(Code::Unimplemented, "fake wire store")) + } +} + +/// A wrapper `StoreDriver` whose `inner_store()` forwards to its wrapped +/// `Store`, unlike most production wrappers (e.g. `CompressionStore`, +/// `VerifyStore`) which return `self`. It proves wire-compression capability +/// discovery remains deliberately immediate. +#[derive(Debug, MetricsComponent)] +struct ForwardingWrapperStore { + inner: Store, +} + +#[async_trait] +#[allow(clippy::todo)] +impl StoreDriver for ForwardingWrapperStore { + async fn post_init(self: Arc) -> Result<(), Error> { + Ok(()) + } + + async fn has_with_results( + self: Pin<&Self>, + _keys: &[StoreKey<'_>], + _results: &mut [Option], ) -> Result<(), Error> { todo!(); } + + async fn update( + self: Pin<&Self>, + _key: StoreKey<'_>, + _reader: DropCloserReadHalf, + _size_info: UploadSizeInfo, + ) -> Result { + todo!(); + } + + async fn get_part( + self: Pin<&Self>, + _key: StoreKey<'_>, + _writer: &mut DropCloserWriteHalf, + _offset: u64, + _length: Option, + ) -> Result<(), Error> { + todo!(); + } + + fn inner_store(&self, digest: Option) -> &dyn StoreDriver { + self.inner.inner_store(digest) + } + + fn as_any(&self) -> &(dyn core::any::Any + Sync + Send + 'static) { + self + } + + fn as_any_arc(self: Arc) -> Arc { + self + } + + fn register_remove_callback(self: Arc, _callback: RemoveCallback) -> Result<(), Error> { + todo!(); + } } -default_health_status_indicator!(FakeStore); +default_health_status_indicator!(ForwardingWrapperStore); + +#[nativelink_test] +async fn wire_compression_capability_only_matches_outer_driver() -> Result<(), Error> { + let immediate_store = Store::new(Arc::new(FakeStore {})); + assert!(immediate_store.wire_compression_store().is_some()); + + let wrapped_store = Store::new(Arc::new(ForwardingWrapperStore { + inner: immediate_store, + })); + assert!( + wrapped_store.wire_compression_store().is_none(), + "wrappers must not recursively expose a representation-changing store's wire capability" + ); + + Ok(()) +} #[nativelink_test] async fn fast_has_with_results() -> Result<(), Error> { diff --git a/nativelink-util/tests/wire_compression_test.rs b/nativelink-util/tests/wire_compression_test.rs new file mode 100644 index 000000000..935bb2119 --- /dev/null +++ b/nativelink-util/tests/wire_compression_test.rs @@ -0,0 +1,132 @@ +// Copyright 2026 The NativeLink Authors. All rights reserved. +// +// Licensed under the Functional Source License, Version 1.1, Apache 2.0 Future License (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// See LICENSE file for details +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use bytes::Bytes; +use nativelink_error::Error; +use nativelink_macro::nativelink_test; +use nativelink_proto::build::bazel::remote::execution::v2::compressor; +use nativelink_util::buf_channel::make_buf_channel_pair; +use nativelink_util::common::DigestInfo; +use nativelink_util::digest_hasher::{DigestHasher, DigestHasherFunc}; +use nativelink_util::wire_compression::{ + ZSTD_COMPRESSION_LEVEL, stream_decode_compressed_upload, stream_encode_compressed_download, +}; +use pretty_assertions::assert_eq; + +fn make_content_with_digest(tag: u8, len: usize) -> (Bytes, DigestInfo) { + let mut data = vec![0u8; len]; + for (i, b) in data.iter_mut().enumerate() { + #[allow(clippy::cast_possible_truncation)] + let byte = match i % 8 { + 0..=4 => tag, + 5 => (i / 8) as u8, + _ => (i / 4096) as u8, + }; + *b = byte; + } + let mut hasher = DigestHasherFunc::Sha256.hasher(); + hasher.update(&data); + let digest = hasher.finalize_digest(); + (Bytes::from(data), digest) +} + +async fn decode_chunks( + frame: Bytes, + split_at: usize, + digest: DigestInfo, +) -> (Result<(), Error>, Result) { + let (mut compressed_tx, compressed_rx) = make_buf_channel_pair(); + let (decoded_tx, mut decoded_rx) = make_buf_channel_pair(); + let decode_fut = stream_decode_compressed_upload( + compressed_rx, + compressor::Value::Zstd, + digest, + DigestHasherFunc::Sha256, + decoded_tx, + ); + let feed_fut = async move { + compressed_tx.send(frame.slice(0..split_at)).await?; + if split_at < frame.len() { + compressed_tx.send(frame.slice(split_at..)).await?; + } + compressed_tx.send_eof() + }; + let pump_fut = async move { + let mut total = 0usize; + loop { + let chunk = decoded_rx.recv().await?; + if chunk.is_empty() { + return Ok(total); + } + total += chunk.len(); + } + }; + let (feed_result, decode_result, pump_result) = tokio::join!(feed_fut, decode_fut, pump_fut); + feed_result.expect("feed must succeed"); + (decode_result, pump_result) +} + +// Regression: a blob whose decompressed size is an exact multiple of the +// decoder output buffer (128KiB) made the final `run` fill the output +// exactly with `hint == 0` (frame done); the loop then polled the finished +// decoder once more, received the new-frame-header hint, and the EOF check +// misreported the complete stream as truncated. +#[nativelink_test] +async fn decode_accepts_exact_output_buffer_multiple() -> Result<(), Error> { + let (content, digest) = make_content_with_digest(0xB2, 1024 * 1024); + let frame = + Bytes::from(zstd::bulk::compress(&content, ZSTD_COMPRESSION_LEVEL).expect("compress")); + let (decode_result, pump_result) = decode_chunks(frame, 65536, digest).await; + decode_result.expect("exact-multiple frame must decode"); + assert_eq!(pump_result.expect("pump"), content.len()); + Ok(()) +} + +// Encode -> decode round trip through the streaming codecs. +#[nativelink_test] +async fn stream_encode_decode_round_trip() -> Result<(), Error> { + let (content, digest) = make_content_with_digest(0xC4, 300_000); + let (mut raw_tx, raw_rx) = make_buf_channel_pair(); + let (compressed_tx, mut compressed_rx) = make_buf_channel_pair(); + let encode_fut = stream_encode_compressed_download( + raw_rx, + compressor::Value::Zstd, + ZSTD_COMPRESSION_LEVEL, + compressed_tx, + ); + let content_for_send = content.clone(); + let send_fut = async move { + raw_tx.send(content_for_send).await?; + raw_tx.send_eof() + }; + let collect_fut = async move { + let mut frame = Vec::new(); + loop { + let chunk = compressed_rx.recv().await?; + if chunk.is_empty() { + return Ok::<_, Error>(Bytes::from(frame)); + } + frame.extend_from_slice(&chunk); + } + }; + let (send_result, encode_result, frame) = tokio::join!(send_fut, encode_fut, collect_fut); + send_result.expect("send"); + encode_result.expect("encode"); + let frame = frame.expect("collect"); + + let (decode_result, pump_result) = decode_chunks(frame, 1, digest).await; + decode_result.expect("decode"); + assert_eq!(pump_result.expect("pump"), content.len()); + Ok(()) +} diff --git a/nativelink-worker/Cargo.toml b/nativelink-worker/Cargo.toml index 9263e36c1..5da074b6a 100644 --- a/nativelink-worker/Cargo.toml +++ b/nativelink-worker/Cargo.toml @@ -4,7 +4,7 @@ lints.workspace = true [package] edition = "2024" name = "nativelink-worker" -version = "1.6.1" +version = "1.6.3" [features] nix = [] diff --git a/nativelink-worker/src/directory_cache.rs b/nativelink-worker/src/directory_cache.rs index cd8d609e4..544c9e256 100644 --- a/nativelink-worker/src/directory_cache.rs +++ b/nativelink-worker/src/directory_cache.rs @@ -28,16 +28,19 @@ use nativelink_metric::{ MetricFieldData, MetricKind, MetricPublishKnownKindData, MetricsComponent, group, publish, }; use nativelink_proto::build::bazel::remote::execution::v2::{ - Directory as ProtoDirectory, DirectoryNode, FileNode, SymlinkNode, + Directory as ProtoDirectory, DirectoryNode, FileNode, GetTreeRequest, SymlinkNode, }; use nativelink_store::ac_utils::get_and_decode_digest; use nativelink_store::cas_utils::is_zero_digest; use nativelink_store::fast_slow_store::FastSlowStore; use nativelink_store::filesystem_store::{FileEntry, FilesystemStore}; +use nativelink_store::grpc_store::GrpcStore; use nativelink_util::background_spawn; use nativelink_util::common::DigestInfo; +use nativelink_util::digest_hasher::{DigestHasher, DigestHasherFunc, default_digest_hasher_func}; use nativelink_util::fs_util::{CloneMethod, hardlink_directory_tree, set_dir_writable_recursive}; use nativelink_util::store_trait::{StoreKey, StoreLike}; +use prost::Message; use tokio::fs; /// Maximum number of concurrently-polled node materializations (file @@ -101,6 +104,8 @@ pub struct DirectoryCacheConfig { /// see the config-crate doc for the `experimental_read_batching` /// interaction. Must be > 0. Default: 64. pub max_concurrent_fetches: usize, + /// See `nativelink-config`'s `DirectoryCacheConfig::experimental_get_tree_prefetch`. + pub experimental_get_tree_prefetch: bool, } impl Default for DirectoryCacheConfig { @@ -111,6 +116,7 @@ impl Default for DirectoryCacheConfig { cache_root: std::env::temp_dir().join("nativelink_directory_cache"), experimental_subtree_caching: false, max_concurrent_fetches: DEFAULT_MAX_CONCURRENT_FETCHES, + experimental_get_tree_prefetch: false, } } } @@ -399,7 +405,9 @@ impl DirectoryCache { /// * `Ok(false)` - Cache miss (directory was constructed) /// * `Err` - Error during construction or hardlinking pub async fn get_or_create(&self, digest: DigestInfo, dest_path: &Path) -> Result { - let (hit, _size) = self.get_or_create_entry(digest, dest_path).await?; + let (hit, _size) = self + .get_or_create_entry(digest, dest_path, None, true) + .await?; Ok(hit) } @@ -409,10 +417,19 @@ impl DirectoryCache { /// `(hit, size)` where `hit` is true when the destination was served by /// hardlinking an existing cache entry, and `size` is the entry's /// recorded size in bytes. + /// + /// `protos` is a map of prefetched `Directory` protos consulted during + /// construction (see [`Self::prefetch_tree_protos`]); `prefetch_on_miss` + /// is true only for root-level calls, which issue the tree's logical + /// `GetTree` prefetch on a cold miss. Subtree flights pass the root's + /// map down instead, so cache hits (root or subtree) never prefetch. A + /// server may paginate that traversal across multiple RPCs. async fn get_or_create_entry( &self, digest: DigestInfo, dest_path: &Path, + protos: Option<&HashMap>, + prefetch_on_miss: bool, ) -> Result<(bool, u64), Error> { self.maybe_log_summary(); @@ -446,7 +463,9 @@ impl DirectoryCache { // outcome so it cannot grow unbounded. The guard (`_guard`) is still // held until the end of this function — `forget_construction_lock` // only unmaps the Arc; any waiter already cloned it before blocking. - let result = self.construct_and_materialize(digest, dest_path).await; + let result = self + .construct_and_materialize(digest, dest_path, protos, prefetch_on_miss) + .await; self.forget_construction_lock(&digest).await; result } @@ -499,10 +518,14 @@ impl DirectoryCache { /// The cache-miss body, run while holding the per-digest construction /// guard. Split out so `get_or_create_entry` can unconditionally clean up /// the construction-lock map entry afterwards on every exit path. + /// `protos` and `prefetch_on_miss` are documented on + /// [`Self::get_or_create_entry`]. async fn construct_and_materialize( &self, digest: DigestInfo, dest_path: &Path, + protos: Option<&HashMap>, + prefetch_on_miss: bool, ) -> Result<(bool, u64), Error> { // Re-check: another task may have just constructed this digest while // we waited on the construction lock. If the entry turns out to be @@ -532,9 +555,19 @@ impl DirectoryCache { // with the CAS and every other in-flight action that hardlinked the // same blob — the inode-corruption bug PR #2347 fixed. let cache_path = self.get_cache_path(&digest); + // Root-level cold miss: prefetch the whole tree's `Directory` protos + // with one logical `GetTree` traversal, following server pagination. + // Subtree flights never prefetch — they inherit the root's map (or + // `None`) through `protos` instead. + let prefetched = if prefetch_on_miss { + Box::pin(self.prefetch_tree_protos(digest)).await + } else { + None + }; + let protos = prefetched.as_ref().or(protos); let temp_path = self.allocate_scratch_path(TEMP_PREFIX); let mut temp_guard = ScratchGuard::new(temp_path.clone()); - let size = match self.construct_directory(digest, &temp_path).await { + let size = match self.construct_directory(digest, &temp_path, protos).await { Ok(size) => size, Err(e) => { temp_guard.disarm(); @@ -739,16 +772,103 @@ impl DirectoryCache { /// /// Each directory's final mode (0o755) is set at creation time, so no /// separate recursive permission pass is needed after construction. + /// Prefetches every `Directory` proto of `root`'s tree with a logical + /// `GetTree` traversal, following `next_page_token` when the server + /// paginates it, and keys each proto by its re-computed digest (the + /// responses carry protos, not digests). Returns `None` — meaning + /// "use the per-level fetch path" — when the feature is disabled, the + /// slow tier is not a `GrpcStore`, or the stream fails; a `Some` map + /// may also be incomplete, which `construct_directory` tolerates by + /// fetching any missing proto individually. One fetch permit covers + /// the entire paginated traversal. + async fn prefetch_tree_protos( + &self, + root: DigestInfo, + ) -> Option> { + if !self.config.experimental_get_tree_prefetch { + return None; + } + let grpc_store = self + .cas_store + .slow_store() + .downcast_ref::(None)?; + let digest_hasher = opentelemetry::Context::current() + .get::() + .map_or_else(default_digest_hasher_func, |v| *v); + let result: Result, Error> = async { + let _permit = self.acquire_fetch_permit().await?; + let mut protos = HashMap::new(); + let mut page_token = String::new(); + loop { + let mut stream = grpc_store + .get_tree(tonic::Request::new(GetTreeRequest { + instance_name: String::new(), + root_digest: Some(root.into()), + page_size: 0, + page_token, + digest_function: digest_hasher.proto_digest_func().into(), + })) + .await + .err_tip(|| "in prefetch_tree_protos")? + .into_inner(); + let mut next_page_token = String::new(); + while let Some(response) = stream + .message() + .await + .map_err(Error::from) + .err_tip(|| "reading GetTree stream in prefetch_tree_protos")? + { + next_page_token = response.next_page_token; + for directory in response.directories { + // GetTree yields protos without digests; recompute with + // the caller's digest function so lookups match the + // digests embedded in parent directories. + let encoded = directory.encode_to_vec(); + let mut hasher = digest_hasher.hasher(); + hasher.update(&encoded); + protos.insert(hasher.finalize_digest(), directory); + } + } + if next_page_token.is_empty() { + break; + } + page_token = next_page_token; + } + Ok(protos) + } + .await; + match result { + Ok(protos) => { + trace!(?root, protos = protos.len(), "GetTree prefetch complete"); + Some(protos) + } + Err(err) => { + debug!( + ?err, + ?root, + "GetTree prefetch failed; using per-level fetches" + ); + None + } + } + } + fn construct_directory<'a>( &'a self, digest: DigestInfo, dest_path: &'a Path, + protos: Option<&'a HashMap>, ) -> Pin> + Send + 'a>> { Box::pin(async move { debug!(?digest, ?dest_path, "Constructing directory"); - // Fetch the Directory proto (permit held only for the fetch). - let directory: ProtoDirectory = { + // Use the prefetched proto when available; otherwise fetch it + // (permit held only for the fetch). A prefetch-map miss (e.g. + // an incomplete GetTree response) degrades to the fetch path. + let prefetched = protos.and_then(|map| map.get(&digest)); + let directory: ProtoDirectory = if let Some(directory) = prefetched { + directory.clone() + } else { let _permit = self.acquire_fetch_permit().await?; get_and_decode_digest(self.cas_store.as_ref(), digest.into()) .await @@ -785,7 +905,9 @@ impl DirectoryCache { })); } for dir_node in &directory.directories { - node_futures.push(Box::pin(self.create_subdirectory(dest_path, dir_node))); + node_futures.push(Box::pin( + self.create_subdirectory(dest_path, dir_node, protos), + )); } for symlink in &directory.symlinks { node_futures.push(Box::pin(async move { @@ -988,6 +1110,7 @@ impl DirectoryCache { &self, parent: &Path, dir_node: &DirectoryNode, + protos: Option<&HashMap>, ) -> Result { let dir_path = parent.join(&dir_node.name); let digest = @@ -1006,7 +1129,9 @@ impl DirectoryCache { // byte-identical wherever else it appears; construction recurses // through this method, so nested subtrees get their own entries // too (leaf-level reuse). - let (hit, _size) = self.get_or_create_entry(digest, &dir_path).await?; + let (hit, _size) = self + .get_or_create_entry(digest, &dir_path, protos, false) + .await?; let counter = if hit { &self.subtree_hits } else { @@ -1025,7 +1150,7 @@ impl DirectoryCache { } // Recursively construct subdirectory - self.construct_directory(digest, &dir_path).await + self.construct_directory(digest, &dir_path, protos).await } /// Creates a symlink from a `SymlinkNode` @@ -2119,4 +2244,26 @@ mod tests { Ok(()) } + + #[nativelink_test] + async fn get_tree_prefetch_falls_back_without_grpc_store() -> Result<(), Error> { + // With the flag on but a non-grpc slow tier, prefetch must return + // None and construction must fall back to per-level fetches with + // identical results. + let temp_dir = TempDir::new().unwrap(); + let cache_root = temp_dir.path().join("cache"); + let (store, dir_digest) = setup_test_store(&temp_dir).await; + let config = DirectoryCacheConfig { + max_entries: 10, + max_size_bytes: 1024 * 1024, + cache_root, + experimental_get_tree_prefetch: true, + ..Default::default() + }; + let cache = DirectoryCache::new(config, store).await?; + let dest = temp_dir.path().join("dest"); + assert!(!cache.get_or_create(dir_digest, &dest).await?); + assert!(dest.join("test.txt").exists()); + Ok(()) + } } diff --git a/nativelink-worker/src/local_worker.rs b/nativelink-worker/src/local_worker.rs index a4fe64e47..e91020247 100644 --- a/nativelink-worker/src/local_worker.rs +++ b/nativelink-worker/src/local_worker.rs @@ -629,6 +629,7 @@ pub async fn new_local_worker( cache_root, experimental_subtree_caching: cache_config.experimental_subtree_caching, max_concurrent_fetches: cache_config.max_concurrent_fetches, + experimental_get_tree_prefetch: cache_config.experimental_get_tree_prefetch, }; match DirectoryCache::new(worker_cache_config, fast_slow_store.clone()).await { diff --git a/nativelink-worker/src/namespace_utils.rs b/nativelink-worker/src/namespace_utils.rs index 8ebb1c55b..74b8ec392 100644 --- a/nativelink-worker/src/namespace_utils.rs +++ b/nativelink-worker/src/namespace_utils.rs @@ -43,9 +43,15 @@ impl MaybeNamespacedChild { // self.child.std_child().send_signal(Signal::SIGTERM)?; // return self.child.wait().await.map(|_| ()); if let Some(pid) = self.child.id() { + let pid_t: libc::pid_t = pid.try_into().map_err(|e| { + Error::new( + std::io::ErrorKind::InvalidInput, + format!("pid larger than pid_t type ({pid}): {e}"), + ) + })?; // SAFETY: pid is valid as provided by the wrapper and we are // sending a signal to the namespaced stub. - unsafe { libc::kill(pid as libc::pid_t, libc::SIGTERM) }; + unsafe { libc::kill(pid_t, libc::SIGTERM) }; return self.child.wait().await.map(|_| ()); } } diff --git a/nativelink-worker/src/running_actions_manager.rs b/nativelink-worker/src/running_actions_manager.rs index 6e4b20573..9e8e77208 100644 --- a/nativelink-worker/src/running_actions_manager.rs +++ b/nativelink-worker/src/running_actions_manager.rs @@ -35,9 +35,7 @@ use std::time::SystemTime; use bytes::{Bytes, BytesMut}; use filetime::{FileTime, set_file_mtime}; use formatx::Template; -use futures::future::{ - BoxFuture, Future, FutureExt, TryFutureExt, try_join, try_join_all, try_join3, -}; +use futures::future::{BoxFuture, Future, FutureExt, TryFutureExt, try_join, try_join_all}; use futures::stream::{FuturesUnordered, StreamExt, TryStreamExt}; use nativelink_config::cas_server::{ EnvironmentSource, UploadActionResultConfig, UploadCacheResultsStrategy, @@ -866,9 +864,11 @@ fn upload_directory<'a, P: AsRef + Debug + Send + Sync + Clone + 'a>( } } - let (mut file_nodes, dir_entries, mut symlinks) = try_join3( + let dir_entries = dir_futures + .try_collect::)>>() + .await?; + let (mut file_nodes, mut symlinks) = try_join( file_futures.try_collect::>(), - dir_futures.try_collect::)>>(), symlink_futures.try_collect::>(), ) .await?; @@ -1459,23 +1459,26 @@ impl RunningActionImpl { #[cfg(target_os = "linux")] { let use_namespaces = self.running_actions_manager.use_namespaces; - let root_action_directory = - std::ffi::CString::new(self.running_actions_manager.root_action_directory.clone()) - .err_tip(|| "In RunningActionImpl::inner_execute()")?; - let action_directory = std::ffi::CString::new(self.action_directory.clone()) + + if !matches!(use_namespaces, UseNamespaces::No) { + let root_action_directory = std::ffi::CString::new( + self.running_actions_manager.root_action_directory.clone(), + ) .err_tip(|| "In RunningActionImpl::inner_execute()")?; + let action_directory = std::ffi::CString::new(self.action_directory.clone()) + .err_tip(|| "In RunningActionImpl::inner_execute()")?; - // SAFETY: This function is specifically designed to operate in a async-signal-safe - // environment. - unsafe { - command_builder.pre_exec(move || match use_namespaces { - UseNamespaces::No => Ok(()), - _ => crate::namespace_utils::configure_namespace( - matches!(use_namespaces, UseNamespaces::YesAndMount), - &root_action_directory, - &action_directory, - ), - }); + // SAFETY: This function is specifically designed to operate in a async-signal-safe + // environment. + unsafe { + command_builder.pre_exec(move || { + crate::namespace_utils::configure_namespace( + matches!(use_namespaces, UseNamespaces::YesAndMount), + &root_action_directory, + &action_directory, + ) + }); + } } // Run the action as its own process-group leader (pgid == child diff --git a/nativelink-worker/tests/directory_cache_test.rs b/nativelink-worker/tests/directory_cache_test.rs index 5848d5692..beb85f99c 100644 --- a/nativelink-worker/tests/directory_cache_test.rs +++ b/nativelink-worker/tests/directory_cache_test.rs @@ -21,29 +21,43 @@ use std::sync::Arc; use async_trait::async_trait; use bytes::Bytes; +use futures::Stream; use nativelink_config::stores::{ - FastSlowSpec, FilesystemSpec, MemorySpec, StoreDirection, StoreSpec, + FastSlowSpec, FilesystemSpec, GrpcEndpoint, GrpcSpec, MemorySpec, Retry, StoreDirection, + StoreSpec, StoreType, }; use nativelink_error::Error; use nativelink_macro::nativelink_test; use nativelink_metric::{ MetricFieldData, MetricKind, MetricPublishKnownKindData, MetricsComponent, }; +use nativelink_proto::build::bazel::remote::execution::v2::content_addressable_storage_server::{ + ContentAddressableStorage, ContentAddressableStorageServer, +}; use nativelink_proto::build::bazel::remote::execution::v2::{ - Directory as ProtoDirectory, DirectoryNode, FileNode, SymlinkNode, + BatchReadBlobsRequest, BatchReadBlobsResponse, BatchUpdateBlobsRequest, + BatchUpdateBlobsResponse, Directory as ProtoDirectory, DirectoryNode, FileNode, + FindMissingBlobsRequest, FindMissingBlobsResponse, GetTreeRequest, GetTreeResponse, + SpliceBlobRequest, SpliceBlobResponse, SplitBlobRequest, SplitBlobResponse, SymlinkNode, }; use nativelink_store::fast_slow_store::FastSlowStore; use nativelink_store::filesystem_store::FilesystemStore; +use nativelink_store::grpc_store::GrpcStore; use nativelink_store::memory_store::MemoryStore; +use nativelink_util::background_spawn; use nativelink_util::buf_channel::{DropCloserReadHalf, DropCloserWriteHalf}; use nativelink_util::common::{DigestInfo, make_temp_path}; +use nativelink_util::digest_hasher::{DigestHasher, default_digest_hasher_func}; use nativelink_util::health_utils::{HealthStatusIndicator, default_health_status_indicator}; use nativelink_util::store_trait::{ RemoveItemCallback, Store, StoreDriver, StoreKey, StoreLike, UploadSizeInfo, }; use nativelink_worker::directory_cache::{DirectoryCache, DirectoryCacheConfig}; use prost::Message; -use tonic::Code; +use tokio::sync::Mutex; +use tonic::transport::Server; +use tonic::transport::server::TcpIncoming; +use tonic::{Code, Request, Response, Status}; use uuid::Uuid; /// Wraps a `MemoryStore` as the slow tier of a `FastSlowStore` whose fast @@ -1432,3 +1446,202 @@ async fn subtree_caching_size_accounting_not_depth_multiplied() -> Result<(), Er ); Ok(()) } + +#[derive(Clone)] +struct PaginatedGetTreeServer { + root: ProtoDirectory, + child: ProtoDirectory, + requests: Arc>>, +} + +type PaginatedGetTreeStream = + Pin> + Send + 'static>>; + +#[async_trait] +impl ContentAddressableStorage for PaginatedGetTreeServer { + type GetTreeStream = PaginatedGetTreeStream; + + async fn find_missing_blobs( + &self, + _request: Request, + ) -> Result, Status> { + Err(Status::unimplemented("not used by this test")) + } + + async fn batch_update_blobs( + &self, + _request: Request, + ) -> Result, Status> { + Err(Status::unimplemented("not used by this test")) + } + + async fn batch_read_blobs( + &self, + _request: Request, + ) -> Result, Status> { + Err(Status::unimplemented("not used by this test")) + } + + async fn get_tree( + &self, + request: Request, + ) -> Result, Status> { + const SECOND_PAGE_TOKEN: &str = "child-directory-page"; + + let request = request.into_inner(); + self.requests.lock().await.push(request.clone()); + let response = match request.page_token.as_str() { + "" => GetTreeResponse { + directories: vec![self.root.clone()], + next_page_token: SECOND_PAGE_TOKEN.to_string(), + }, + SECOND_PAGE_TOKEN => GetTreeResponse { + directories: vec![self.child.clone()], + next_page_token: String::new(), + }, + token => { + return Err(Status::invalid_argument(format!( + "unexpected token: {token}" + ))); + } + }; + Ok(Response::new(Box::pin(futures::stream::iter([Ok( + response, + )])))) + } + + async fn split_blob( + &self, + _request: Request, + ) -> Result, Status> { + Err(Status::unimplemented("not used by this test")) + } + + async fn splice_blob( + &self, + _request: Request, + ) -> Result, Status> { + Err(Status::unimplemented("not used by this test")) + } +} + +fn proto_digest(directory: &ProtoDirectory) -> DigestInfo { + let mut hasher = default_digest_hasher_func().hasher(); + hasher.update(&directory.encode_to_vec()); + hasher.finalize_digest() +} + +/// The [REAPI request contract] says a zero page size does not require a +/// server to return the entire tree in one response, and the [REAPI response +/// contract] tells the client to continue with `next_page_token`. This test +/// therefore deliberately uses the smallest tree that cannot be represented +/// by the first page alone. The first page contains only the root `Directory`, +/// whose `DirectoryNode` names and hashes the child, while the second page +/// contains only that child `Directory`. +/// +/// Using two different, content-derived directory digests matters here: the +/// public `get_or_create` operation cannot materialize the named child unless +/// the prefetch map contains both pages. The fake server also records the +/// requests so the test verifies that the continuation token was copied +/// exactly into the follow-up request and that fetching stopped as soon as the +/// server returned an empty token. No directory protos are seeded into the +/// filesystem store, and the server intentionally does not expose a +/// `ByteStream` service. Consequently, ignoring the second page cannot be +/// hidden by the normal per-level fallback: that fallback would fail while +/// fetching the absent child proto. Before the fix, the client drained the first response +/// stream, discarded the advertised child-directory page, and took precisely +/// that failing fallback path. +/// +/// [REAPI request contract]: https://github.com/bazelbuild/remote-apis/blob/becdd8f9ff811df88a22d3eadd6341753d51d167/build/bazel/remote/execution/v2/remote_execution.proto#L1932-L1942 +/// [REAPI response contract]: https://github.com/bazelbuild/remote-apis/blob/becdd8f9ff811df88a22d3eadd6341753d51d167/build/bazel/remote/execution/v2/remote_execution.proto#L1961-L1965 +#[nativelink_test] +async fn get_tree_prefetch_follows_server_pagination() -> Result<(), Error> { + let child = ProtoDirectory::default(); + let child_digest = proto_digest(&child); + let root = ProtoDirectory { + directories: vec![DirectoryNode { + name: "second-page".to_string(), + digest: Some(child_digest.into()), + }], + ..Default::default() + }; + let root_digest = proto_digest(&root); + let requests = Arc::new(Mutex::new(Vec::new())); + let service = PaginatedGetTreeServer { + root, + child, + requests: Arc::clone(&requests), + }; + let listener = TcpIncoming::bind("127.0.0.1:0".parse().unwrap()).unwrap(); + let port = listener.local_addr().unwrap().port(); + background_spawn!("paginated_get_tree_server", async move { + Server::builder() + .add_service(ContentAddressableStorageServer::new(service)) + .serve_with_incoming(listener) + .await + .unwrap(); + }); + + let grpc_spec = GrpcSpec { + instance_name: String::new(), + endpoints: vec![GrpcEndpoint { + address: format!("http://127.0.0.1:{port}"), + tls_config: None, + concurrency_limit: None, + connect_timeout_s: 0, + tcp_keepalive_s: 0, + http2_keepalive_interval_s: 0, + http2_keepalive_timeout_s: 0, + }], + store_type: StoreType::Cas, + retry: Retry::default(), + max_concurrent_requests: 0, + connections_per_endpoint: 0, + rpc_timeout_s: 1, + use_legacy_resource_names: false, + headers: HashMap::new(), + forward_headers: vec![], + experimental_read_batching: None, + experimental_remote_cache_compression: false, + }; + let fast_spec = FilesystemSpec { + content_path: make_temp_path("paginated_get_tree_cas_content"), + temp_path: make_temp_path("paginated_get_tree_cas_temp"), + ..Default::default() + }; + let fast_store: Arc = FilesystemStore::new(&fast_spec).await?; + let grpc_store = GrpcStore::new(&grpc_spec).await?; + let cas_store = FastSlowStore::new( + &FastSlowSpec { + fast: StoreSpec::Filesystem(fast_spec), + slow: StoreSpec::Grpc(grpc_spec), + fast_direction: StoreDirection::default(), + slow_direction: StoreDirection::default(), + bypass_dedup_threshold_bytes: 0, + }, + Store::new(fast_store), + Store::new(grpc_store), + ); + let cache = DirectoryCache::new( + DirectoryCacheConfig { + cache_root: make_temp_path("paginated_get_tree_directory_cache").into(), + experimental_get_tree_prefetch: true, + ..Default::default() + }, + cas_store, + ) + .await?; + + let destination = PathBuf::from(make_temp_path("paginated_get_tree_destination")); + assert!(!cache.get_or_create(root_digest, &destination).await?); + assert!(destination.join("second-page").is_dir()); + + let requests = requests.lock().await; + assert_eq!(requests.len(), 2); + assert_eq!(requests[0].page_token, ""); + assert_eq!(requests[1].page_token, "child-directory-page"); + assert_eq!(requests[0].root_digest, Some(root_digest.into())); + assert_eq!(requests[1].root_digest, Some(root_digest.into())); + + Ok(()) +} diff --git a/nativelink-worker/tests/running_actions_manager_test.rs b/nativelink-worker/tests/running_actions_manager_test.rs index 8768af642..e2007c150 100644 --- a/nativelink-worker/tests/running_actions_manager_test.rs +++ b/nativelink-worker/tests/running_actions_manager_test.rs @@ -645,6 +645,7 @@ mod tests { let execute_request = ExecuteRequest { action_digest: Some(action_digest.into()), + digest_function: ProtoDigestFunction::Sha256.into(), ..Default::default() }; let operation_id = OperationId::default().to_string(); @@ -773,6 +774,7 @@ mod tests { let execute_request = ExecuteRequest { action_digest: Some(action_digest.into()), + digest_function: ProtoDigestFunction::Sha256.into(), ..Default::default() }; let operation_id = OperationId::default().to_string(); @@ -1100,6 +1102,7 @@ mod tests { let execute_request = ExecuteRequest { action_digest: Some(action_digest.into()), + digest_function: ProtoDigestFunction::Sha256.into(), ..Default::default() }; let operation_id = OperationId::default().to_string(); @@ -1280,6 +1283,7 @@ mod tests { let execute_request = ExecuteRequest { action_digest: Some(action_digest.into()), + digest_function: ProtoDigestFunction::Sha256.into(), ..Default::default() }; let operation_id = OperationId::default().to_string(); @@ -1514,6 +1518,7 @@ mod tests { let execute_request = ExecuteRequest { action_digest: Some(action_digest.into()), + digest_function: ProtoDigestFunction::Sha256.into(), ..Default::default() }; let operation_id = OperationId::default().to_string(); @@ -1665,6 +1670,7 @@ mod tests { .await?; let execute_request = ExecuteRequest { action_digest: Some(action_digest.into()), + digest_function: ProtoDigestFunction::Sha256.into(), ..Default::default() }; let operation_id = OperationId::default().to_string(); @@ -1811,6 +1817,7 @@ mod tests { let execute_request = ExecuteRequest { action_digest: Some(action_digest.into()), + digest_function: ProtoDigestFunction::Sha256.into(), ..Default::default() }; let operation_id = OperationId::default().to_string(); @@ -1962,6 +1969,7 @@ mod tests { let execute_request = ExecuteRequest { action_digest: Some(action_digest.into()), + digest_function: ProtoDigestFunction::Sha256.into(), ..Default::default() }; let operation_id = OperationId::default().to_string(); @@ -2145,6 +2153,7 @@ exit 0 let execute_request = ExecuteRequest { action_digest: Some(action_digest.into()), + digest_function: ProtoDigestFunction::Sha256.into(), ..Default::default() }; let operation_id = OperationId::default().to_string(); @@ -2320,7 +2329,7 @@ exit 0 }], }), timeout: Some(prost_types::Duration { - seconds: TASK_TIMEOUT.as_secs() as i64, + seconds: TASK_TIMEOUT.as_secs().try_into().unwrap_or(i64::MAX), nanos: 0, }), ..Default::default() @@ -2334,6 +2343,7 @@ exit 0 let execute_request = ExecuteRequest { action_digest: Some(action_digest.into()), + digest_function: ProtoDigestFunction::Sha256.into(), ..Default::default() }; let operation_id = OperationId::default().to_string(); @@ -2494,6 +2504,7 @@ exit 1 let execute_request = ExecuteRequest { action_digest: Some(action_digest.into()), + digest_function: ProtoDigestFunction::Sha256.into(), ..Default::default() }; let operation_id = OperationId::default().to_string(); @@ -3028,7 +3039,7 @@ exit 1 command_digest: Some(command_digest.into()), input_root_digest: Some(input_root_digest.into()), timeout: Some(prost_types::Duration { - seconds: TASK_TIMEOUT.as_secs() as i64, + seconds: TASK_TIMEOUT.as_secs().try_into().unwrap_or(i64::MAX), nanos: 0, }), ..Default::default() @@ -3074,6 +3085,7 @@ exit 1 let execute_request = ExecuteRequest { action_digest: Some(action_digest.into()), + digest_function: ProtoDigestFunction::Sha256.into(), ..Default::default() }; let operation_id = OperationId::default().to_string(); @@ -3118,7 +3130,7 @@ exit 1 command_digest: Some(command_digest.into()), input_root_digest: Some(input_root_digest.into()), timeout: Some(prost_types::Duration { - seconds: TASK_TIMEOUT.as_secs() as i64, + seconds: TASK_TIMEOUT.as_secs().try_into().unwrap_or(i64::MAX), nanos: 0, }), ..Default::default() @@ -3164,6 +3176,7 @@ exit 1 let execute_request = ExecuteRequest { action_digest: Some(action_digest.into()), + digest_function: ProtoDigestFunction::Sha256.into(), ..Default::default() }; let operation_id = OperationId::default().to_string(); @@ -3208,7 +3221,7 @@ exit 1 command_digest: Some(command_digest.into()), input_root_digest: Some(input_root_digest.into()), timeout: Some(prost_types::Duration { - seconds: TASK_TIMEOUT.as_secs() as i64, + seconds: TASK_TIMEOUT.as_secs().try_into().unwrap_or(i64::MAX), nanos: 0, }), ..Default::default() @@ -3254,6 +3267,7 @@ exit 1 let execute_request = ExecuteRequest { action_digest: Some(action_digest.into()), + digest_function: ProtoDigestFunction::Sha256.into(), ..Default::default() }; let operation_id = OperationId::default().to_string(); @@ -3385,6 +3399,7 @@ exit 1 let execute_request = ExecuteRequest { action_digest: Some(action_digest.into()), + digest_function: ProtoDigestFunction::Sha256.into(), ..Default::default() }; let operation_id = OperationId::default().to_string(); @@ -3531,6 +3546,7 @@ exit 1 let execute_request = ExecuteRequest { action_digest: Some(action_digest.into()), + digest_function: ProtoDigestFunction::Sha256.into(), ..Default::default() }; let operation_id = OperationId::default().to_string(); @@ -3694,6 +3710,7 @@ exit 1 let execute_request = ExecuteRequest { action_digest: Some(action_digest.into()), + digest_function: ProtoDigestFunction::Sha256.into(), ..Default::default() }; let operation_id = OperationId::default().to_string(); @@ -3719,6 +3736,129 @@ exit 1 Ok(()) } + /// Regression for skipping the `pre_exec` hook when namespaces are off. + /// With namespaces disabled (the default) the action spawn goes through + /// `posix_spawn` instead of `fork`, and `process_group(0)` must still make + /// the child its own process-group leader (pgid == pid). + #[cfg(target_os = "linux")] + #[nativelink_test] + async fn no_namespace_action_is_process_group_leader() -> Result<(), Box> + { + const WORKER_ID: &str = "foo_worker_id"; + + fn test_monotonic_clock() -> SystemTime { + static CLOCK: AtomicU64 = AtomicU64::new(0); + monotonic_clock(&CLOCK) + } + + let (_, _, cas_store, ac_store) = setup_stores().await?; + let root_action_directory = make_temp_path("root_action_directory"); + fs::create_dir_all(&root_action_directory).await?; + + let running_actions_manager = Arc::new(RunningActionsManagerImpl::new_with_callbacks( + RunningActionsManagerArgs { + root_action_directory, + cas_store: cas_store.clone(), + ac_store: Some(Store::new(ac_store.clone())), + execution_configuration: ExecutionConfiguration::default(), + historical_store: Store::new(cas_store.clone()), + upload_action_result_config: &UploadActionResultConfig { + upload_ac_results_strategy: UploadCacheResultsStrategy::Never, + ..Default::default() + }, + max_action_timeout: Duration::MAX, + max_upload_timeout: Duration::from_secs(DEFAULT_MAX_UPLOAD_TIMEOUT), + max_cleanup_wait: Duration::from_secs(DEFAULT_MAX_CLEANUP_WAIT), + max_cleanup_backoff: Duration::from_millis(DEFAULT_MAX_CLEANUP_BACKOFF), + timeout_handled_externally: false, + directory_cache: None, + // Pin namespaces off so this exercises the no-pre_exec/posix_spawn + // path regardless of what the host kernel supports. + use_namespaces: nativelink_worker::running_actions_manager::UseNamespaces::No, + }, + Callbacks { + now_fn: test_monotonic_clock, + sleep_fn: |_duration| Box::pin(future::pending()), + }, + )?); + + // Print the shell's own pid (field 1) and process-group id (field 5) + // from its /proc stat line; process_group(0) makes them equal. + let command = Command { + arguments: vec![ + "sh".to_string(), + "-c".to_string(), + "read -r pid _ _ _ pgrp _ < /proc/$$/stat; printf '%s %s' \"$pid\" \"$pgrp\"" + .to_string(), + ], + output_paths: vec![], + working_directory: ".".to_string(), + environment_variables: vec![EnvironmentVariable { + name: "PATH".to_string(), + value: env::var("PATH").unwrap(), + }], + ..Default::default() + }; + let command_digest = serialize_and_upload_message( + &command, + cas_store.as_pin(), + &mut DigestHasherFunc::Sha256.hasher(), + ) + .await?; + let input_root_digest = serialize_and_upload_message( + &Directory::default(), + cas_store.as_pin(), + &mut DigestHasherFunc::Sha256.hasher(), + ) + .await?; + let action = Action { + command_digest: Some(command_digest.into()), + input_root_digest: Some(input_root_digest.into()), + ..Default::default() + }; + let action_digest = serialize_and_upload_message( + &action, + cas_store.as_pin(), + &mut DigestHasherFunc::Sha256.hasher(), + ) + .await?; + + let execute_request = ExecuteRequest { + action_digest: Some(action_digest.into()), + digest_function: ProtoDigestFunction::Sha256.into(), + ..Default::default() + }; + let operation_id = OperationId::default().to_string(); + let running_action_impl = running_actions_manager + .create_and_add_action( + WORKER_ID.to_string(), + StartExecute { + execute_request: Some(execute_request), + operation_id, + ..Default::default() + }, + ) + .await?; + + let action_result = run_action(running_action_impl.clone()).await?; + assert_eq!( + action_result.exit_code, 0, + "action should run to completion via posix_spawn" + ); + + let stdout = cas_store + .as_ref() + .get_part_unchunked(action_result.stdout_digest, 0, None) + .await?; + let stdout = from_utf8(&stdout)?; + let (pid, pgrp) = stdout.split_once(' ').expect("expected 'pid pgrp' output"); + assert_eq!( + pid, pgrp, + "spawned process should be its own process-group leader (pid={pid} pgrp={pgrp})" + ); + Ok(()) + } + #[nativelink_test] async fn action_directory_contents_are_cleaned() -> Result<(), Box> { const WORKER_ID: &str = "foo_worker_id"; @@ -3792,6 +3932,7 @@ exit 1 let execute_request = ExecuteRequest { action_digest: Some(action_digest.into()), + digest_function: ProtoDigestFunction::Sha256.into(), ..Default::default() }; let operation_id = OperationId::default().to_string(); @@ -3910,6 +4051,7 @@ exit 1 StartExecute { execute_request: Some(ExecuteRequest { action_digest: Some(action_digest.into()), + digest_function: ProtoDigestFunction::Sha256.into(), ..Default::default() }), operation_id: operation_id.to_string(), @@ -4118,6 +4260,7 @@ done let execute_request = ExecuteRequest { action_digest: Some(action_digest.into()), + digest_function: ProtoDigestFunction::Sha256.into(), ..Default::default() }; let operation_id = OperationId::default().to_string(); @@ -4302,6 +4445,7 @@ done let execute_request = ExecuteRequest { action_digest: Some(action_digest.into()), + digest_function: ProtoDigestFunction::Sha256.into(), ..Default::default() }; let operation_id = OperationId::default().to_string(); @@ -4393,6 +4537,7 @@ done let execute_request = ExecuteRequest { action_digest: Some(action_digest.into()), + digest_function: ProtoDigestFunction::Sha256.into(), ..Default::default() }; @@ -4537,6 +4682,7 @@ done let execute_request = ExecuteRequest { action_digest: Some(action_digest.into()), + digest_function: ProtoDigestFunction::Sha256.into(), ..Default::default() }; @@ -4670,6 +4816,7 @@ done StartExecute { execute_request: Some(ExecuteRequest { action_digest: Some(action_digest.into()), + digest_function: ProtoDigestFunction::Sha256.into(), ..Default::default() }), operation_id, @@ -4823,6 +4970,7 @@ done StartExecute { execute_request: Some(ExecuteRequest { action_digest: Some(action_digest.into()), + digest_function: ProtoDigestFunction::Sha256.into(), ..Default::default() }), operation_id: OperationId::default().to_string(), @@ -4939,6 +5087,7 @@ done let execute_request = ExecuteRequest { action_digest: Some(action_digest.into()), + digest_function: ProtoDigestFunction::Sha256.into(), ..Default::default() }; let operation_id = OperationId::default().to_string(); @@ -4998,4 +5147,155 @@ done assert_eq!(parse_pgid_from_stat("123 (only) S"), None); // too few fields assert_eq!(parse_pgid_from_stat(""), None); } + + // Regression test for #2636: deeply nested directories with a single + // semaphore permit previously deadlocked because dir_futures and + // file_futures competed for the same permit inside try_join3. + // The fix awaits dir_futures first so permits are released before + // file/symlink uploads begin. + + #[nativelink_test] + #[cfg(target_family = "unix")] + async fn upload_with_single_permit_nested_dirs() -> Result<(), Box> { + const WORKER_ID: &str = "foo_worker_id"; + + fn test_monotonic_clock() -> SystemTime { + static CLOCK: AtomicU64 = AtomicU64::new(0); + monotonic_clock(&CLOCK) + } + + let (_, _, cas_store, ac_store) = setup_stores().await?; + let root_action_directory = make_temp_path("root_action_directory"); + fs::create_dir_all(&root_action_directory).await?; + + // Take all but one FD permit away to trigger the deadlock scenario. + let _permits = stream::iter(1..fs::OPEN_FILE_SEMAPHORE.available_permits()) + .then(|_| fs::OPEN_FILE_SEMAPHORE.acquire()) + .try_collect::>() + .await?; + assert_eq!(1, fs::OPEN_FILE_SEMAPHORE.available_permits()); + + let running_actions_manager = Arc::new(RunningActionsManagerImpl::new_with_callbacks( + RunningActionsManagerArgs { + root_action_directory, + execution_configuration: ExecutionConfiguration::default(), + cas_store: cas_store.clone(), + ac_store: Some(Store::new(ac_store.clone())), + historical_store: Store::new(cas_store.clone()), + upload_action_result_config: &UploadActionResultConfig { + upload_ac_results_strategy: UploadCacheResultsStrategy::Never, + ..Default::default() + }, + max_action_timeout: Duration::MAX, + max_upload_timeout: Duration::from_secs(DEFAULT_MAX_UPLOAD_TIMEOUT), + max_cleanup_wait: Duration::from_secs(DEFAULT_MAX_CLEANUP_WAIT), + max_cleanup_backoff: Duration::from_millis(DEFAULT_MAX_CLEANUP_BACKOFF), + timeout_handled_externally: false, + directory_cache: None, + #[cfg(target_os = "linux")] + use_namespaces: use_namespaces(), + }, + Callbacks { + now_fn: test_monotonic_clock, + sleep_fn: |_duration| Box::pin(future::pending()), + }, + )?); + + // Create 3-level nested dirs: out/a/b/ with files at each level. + // This exercises recursive upload_directory under permit starvation. + let arguments = vec![ + "sh".to_string(), + "-c".to_string(), + concat!( + "mkdir -p ./out/a/b && ", + "printf 'root ' > ./out/root.txt && ", + "printf 'mid ' > ./out/a/mid.txt && ", + "printf 'leaf ' > ./out/a/b/leaf.txt && ", + "printf 'ok-stdout '; >&2 printf 'ok-stderr '" + ) + .to_string(), + ]; + let working_directory = "some_cwd"; + let command = Command { + arguments, + output_paths: vec!["out".to_string()], + working_directory: working_directory.to_string(), + environment_variables: vec![EnvironmentVariable { + name: "PATH".to_string(), + value: env::var("PATH").unwrap(), + }], + ..Default::default() + }; + let command_digest = serialize_and_upload_message( + &command, + cas_store.as_pin(), + &mut DigestHasherFunc::Sha256.hasher(), + ) + .await?; + let input_root_digest = serialize_and_upload_message( + &Directory { + directories: vec![DirectoryNode { + name: working_directory.to_string(), + digest: Some( + serialize_and_upload_message( + &Directory::default(), + cas_store.as_pin(), + &mut DigestHasherFunc::Sha256.hasher(), + ) + .await? + .into(), + ), + }], + ..Default::default() + }, + cas_store.as_pin(), + &mut DigestHasherFunc::Sha256.hasher(), + ) + .await?; + let action = Action { + command_digest: Some(command_digest.into()), + input_root_digest: Some(input_root_digest.into()), + ..Default::default() + }; + let action_digest = serialize_and_upload_message( + &action, + cas_store.as_pin(), + &mut DigestHasherFunc::Sha256.hasher(), + ) + .await?; + + let execute_request = ExecuteRequest { + action_digest: Some(action_digest.into()), + digest_function: ProtoDigestFunction::Sha256.into(), + ..Default::default() + }; + let operation_id = OperationId::default().to_string(); + + let running_action_impl = running_actions_manager + .create_and_add_action( + WORKER_ID.to_string(), + StartExecute { + execute_request: Some(execute_request), + operation_id, + queued_timestamp: None, + platform: action.platform.clone(), + worker_id: WORKER_ID.to_string(), + }, + ) + .await?; + + // This would deadlock before the fix in #2636 because nested + // dir_futures and file_futures would compete for the single permit. + let action_result = run_action(running_action_impl.clone()).await?; + + assert_eq!(action_result.exit_code, 0, "action should succeed"); + // Verify the nested directory tree was uploaded. + assert_eq!( + action_result.output_folders.len(), + 1, + "expected one output directory" + ); + assert_eq!(action_result.output_folders[0].path, "out"); + Ok(()) + } } diff --git a/src/bin/redis_store_tester.rs b/src/bin/redis_store_tester.rs index b5048161f..4d63757d2 100644 --- a/src/bin/redis_store_tester.rs +++ b/src/bin/redis_store_tester.rs @@ -1,3 +1,4 @@ +use core::pin::Pin; use core::sync::atomic::{AtomicUsize, Ordering}; use core::time::Duration; use std::borrow::Cow; @@ -12,7 +13,7 @@ use nativelink_error::{Code, Error, ResultExt}; use nativelink_store::redis_store::RedisStore; use nativelink_util::buf_channel::make_buf_channel_pair; use nativelink_util::store_trait::{ - SchedulerCurrentVersionProvider, SchedulerIndexProvider, SchedulerStore, + RemoveItemCallback, SchedulerCurrentVersionProvider, SchedulerIndexProvider, SchedulerStore, SchedulerStoreDataProvider, SchedulerStoreDecodeTo, SchedulerStoreKeyProvider, StoreDriver, StoreKey, StoreLike, TrueValue, UploadSizeInfo, }; @@ -155,6 +156,11 @@ async fn run( ) -> Result<(), Error> { let mut count = 0; let in_flight = Arc::new(AtomicUsize::new(0)); + let fixed_action_value = if let Ok(str_action_value) = env::var("ACTION_VALUE") { + Some(str::parse::(&str_action_value).unwrap()) + } else { + None + }; loop { if count % 1000 == 0 { @@ -187,9 +193,13 @@ async fn run( let local_in_flight = in_flight.clone(); let max_action_value = 7; - let action_value = match mode { - TestMode::Random => rand::rng().random_range(0..max_action_value), - TestMode::Sequential => count % max_action_value, + let action_value = if let Some(av) = fixed_action_value { + av + } else { + match mode { + TestMode::Random => rand::rng().random_range(0..max_action_value), + TestMode::Sequential => count % max_action_value, + } }; background_spawn!("action", async move { @@ -222,9 +232,11 @@ async fn run( .await?; } 3 => { + let key = random_key(); store_clone - .update_oneshot(random_key(), Bytes::from_static(b"1234")) + .update_oneshot(key.borrow(), Bytes::from_static(b"1234")) .await?; + info!(?key, "Updated"); } 4 => { let res = store_clone @@ -284,6 +296,19 @@ async fn run( } } +#[derive(Debug)] +struct LoggingRemoveCallback {} + +impl RemoveItemCallback for LoggingRemoveCallback { + fn callback<'a>( + &'a self, + store_key: StoreKey<'a>, + ) -> Pin + Send + 'a>> { + info!(?store_key, "Callback for removed item"); + Box::pin(async {}) + } +} + fn main() -> Result<(), Box> { let args = Args::parse(); let redis_mode: RedisMode = args.redis_mode.into(); @@ -330,10 +355,16 @@ fn main() -> Result<(), Box> { match spec.mode { RedisMode::Standard | RedisMode::Sentinel => { let store = RedisStore::new_standard(spec).await?; + store + .clone() + .register_remove_callback(Arc::new(LoggingRemoveCallback {}))?; run(store, max_loops, failed.clone(), args.mode).await } RedisMode::Cluster => { let store = RedisStore::new_cluster(spec).await?; + store + .clone() + .register_remove_callback(Arc::new(LoggingRemoveCallback {}))?; run(store, max_loops, failed.clone(), args.mode).await } } diff --git a/templates/README.md b/templates/README.md index 2ec132444..56a0ec268 100644 --- a/templates/README.md +++ b/templates/README.md @@ -10,7 +10,7 @@ NativeLink provides the following templates to use caching and remote execution: C/C++ with CMake using [`recc`](https://buildgrid.gitlab.io/recc) as the bridge to NativeLink. Cache-only by default. Compiles run locally and only their outputs travel through the cache. Works on Linux and macOS. - Tutorial: [Build CMake projects with NativeLink](https://nativelink.com/docs/rbe/cmake-recc). + Tutorial: [Build CMake projects with NativeLink](https://docs.nativelink.com/getting-started/other-build-systems/cmake-recc). # Getting started diff --git a/templates/bazel/.github/workflows/lre.yaml b/templates/bazel/.github/workflows/lre.yaml index ef5e4eca2..7a81115c7 100644 --- a/templates/bazel/.github/workflows/lre.yaml +++ b/templates/bazel/.github/workflows/lre.yaml @@ -34,4 +34,4 @@ jobs: source-tag: v3.13.0 - name: Build project - run: nix develop --command bazel build ... + run: nix develop --command bazel-retry build ... diff --git a/templates/cmake/README.md b/templates/cmake/README.md index 13d18b1b2..272399627 100644 --- a/templates/cmake/README.md +++ b/templates/cmake/README.md @@ -1,7 +1,7 @@ # CMake template The full tutorial lives at -[Build CMake projects with NativeLink](https://nativelink.com/docs/rbe/cmake-recc). +[Build CMake projects with NativeLink](https://docs.nativelink.com/getting-started/other-build-systems/cmake-recc). This directory holds the example sources (`CMakeLists.txt`, `main.cpp`) that the tutorial references. Copy them into a fresh project directory diff --git a/toolchain-examples/MODULE.bazel b/toolchain-examples/MODULE.bazel index c37339476..90fbe2cf3 100644 --- a/toolchain-examples/MODULE.bazel +++ b/toolchain-examples/MODULE.bazel @@ -7,7 +7,7 @@ module( bazel_dep(name = "platforms", version = "1.1.0") # C++ -bazel_dep(name = "rules_cc", version = "0.2.18") +bazel_dep(name = "rules_cc", version = "0.2.19") # Java bazel_dep(name = "rules_java", version = "9.0.3") @@ -38,8 +38,8 @@ bazel_dep(name = "rules_go", version = "0.60.0") # Rust # -bazel_dep(name = "rules_rs", version = "0.0.76") -bazel_dep(name = "llvm", version = "0.7.7") +bazel_dep(name = "rules_rs", version = "0.0.96") +bazel_dep(name = "llvm", version = "0.8.9") rust_toolchains = use_extension("@rules_rs//rs/toolchains:module_extension.bzl", "toolchains") rust_toolchains.toolchain( diff --git a/tools/cargo-llvm-cov/Cargo.lock b/tools/cargo-llvm-cov/Cargo.lock index 40fa992bb..c8499e41a 100644 --- a/tools/cargo-llvm-cov/Cargo.lock +++ b/tools/cargo-llvm-cov/Cargo.lock @@ -13,9 +13,9 @@ dependencies = [ [[package]] name = "anyhow" -version = "1.0.102" +version = "1.0.103" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f202df86484c868dbad7eaa557ef785d5c66295e41b460ef922eca0723b842c" +checksum = "2a4385e2e34eb35d6b3efe798b9eb88096925d87726c0798709bf56d9ed84af3" [[package]] name = "autocfg" diff --git a/tools/cargo-llvm-cov/osv-scanner.toml b/tools/cargo-llvm-cov/osv-scanner.toml new file mode 100644 index 000000000..143a6e642 --- /dev/null +++ b/tools/cargo-llvm-cov/osv-scanner.toml @@ -0,0 +1,9 @@ +# Ignore false positives as per https://github.com/taiki-e/cargo-llvm-cov/blob/main/.deny.toml + +[[IgnoredVulns]] +id = "RUSTSEC-2026-0194" +reason = "quick-xml (via lcov2cobertura). Not affected since lcov2cobertura emits xml, but doesn't parse." + +[[IgnoredVulns]] +id = "RUSTSEC-2026-0195" +reason = "quick-xml (via lcov2cobertura). Not affected since lcov2cobertura doesn't use affected APIs." diff --git a/tools/generate-bazel-rc/src/main.rs b/tools/generate-bazel-rc/src/main.rs index a0d54021b..1debc5285 100644 --- a/tools/generate-bazel-rc/src/main.rs +++ b/tools/generate-bazel-rc/src/main.rs @@ -76,7 +76,7 @@ fn get_lints_from_key(lints_table: &Map, key: &str) -> BTreeSet + + Each store's `eviction_policy.max_bytes` caps it; once full, the + least-recently-used entries are evicted to make room. An evicted + entry is just a future cache miss — clients rebuild and re-upload. + + + The scheduler's `supported_platform_properties` must be satisfiable + by at least one worker's `platform_properties`, or matching actions + queue indefinitely with no error surfaced. Compare those two blocks + first. + + + When more than one node needs the same cache, or when a restart + wiping state starts to hurt. That's the cue for the durable, sharded + shapes in [Production configurations](/configuration/production). + + + ## What's next - [Production configurations](/configuration/production) — the diff --git a/web/apps/docs/content/docs/configuration/chunking.mdx b/web/apps/docs/content/docs/configuration/chunking.mdx index 605284ed7..bfa28bda1 100644 --- a/web/apps/docs/content/docs/configuration/chunking.mdx +++ b/web/apps/docs/content/docs/configuration/chunking.mdx @@ -108,3 +108,26 @@ materializes the full blob, so non-chunking clients and every existing read path see ordinary blobs. Storage grows by roughly the chunk bytes for chunk-eligible blobs; pairing the CAS with a `dedup` or `compression` store composes normally. + +## FAQ + + + + No. Chunking is opt-in and advertised through the capabilities + service — clients that don't request it (or predate it) keep using + regular transfers against the same CAS. + + + Bazel 9.1.0 has a client bug that corrupts outputs when the chunking + flag is combined with `--disk_cache`. Use 9.1.1+ (or 8.7.0+). + + + They solve different transfer problems. Chunking re-transfers only + the changed parts of large, incrementally-changing blobs; + [wire compression](/configuration/compression) shrinks compressible + blobs wholesale. Compressed artifacts chunk poorly — everything after + the first changed byte re-transfers — so lean on chunking for + archives, binaries, and layers, and on compression for text-heavy + outputs. + + diff --git a/web/apps/docs/content/docs/configuration/compression.mdx b/web/apps/docs/content/docs/configuration/compression.mdx index 28389a8c3..d70642ada 100644 --- a/web/apps/docs/content/docs/configuration/compression.mdx +++ b/web/apps/docs/content/docs/configuration/compression.mdx @@ -88,3 +88,37 @@ Wire compression doesn't change what's stored: the CAS holds raw bytes. To also shrink bytes at rest, wrap the backing store in a [`compression` store](/reference/nativelink-config/main#compressionspec); the two compose, since one covers the wire and the other covers storage. + +## Skipping the decompress/recompress round trip: `compression_algorithm.zstd` + +`compression_algorithm: { lz4: ... }` always decompresses to raw bytes at +the store boundary, so a wire-compressed request still pays a +decompress-on-write and recompress-on-read even though the bytes were zstd on +the wire the whole time. `compression_algorithm: { zstd: ... }` closes that +gap: it keeps blobs as zstd **at rest** and, when it's the outermost store an +instance points at, serves them +byte-for-byte to `--remote_cache_compression` clients — no recompress on a +hit. Identity clients still get plain decompressed bytes. + +It has real deployment constraints — a dedicated, empty backend namespace, +no in-place migration, and a capped `compression_level` — that don't apply +to `compression_algorithm: { lz4: ... }`. See +[Store overview → Pass-through compression](/reference/nativelink-config/store-overview#pass-through-compression-compression_algorithmzstd) +for the full picture before adopting it. + +## FAQ + + + + Yes. Clients only compress when the server advertises support, so + anything that doesn't request `compressed-blobs/zstd` keeps using + identity transfers against the same instance. + + + Yes, but order matters at the store level: deduplication must operate + on raw content — identical chunks stop being identical after + compression, so a `dedup` store fed compressed bytes finds no + duplicates. Wire compression is fine either way; it's undone at the + gRPC boundary before any store sees the bytes. + + diff --git a/web/apps/docs/content/docs/configuration/intro.mdx b/web/apps/docs/content/docs/configuration/intro.mdx index 75fde30dd..058d2c60c 100644 --- a/web/apps/docs/content/docs/configuration/intro.mdx +++ b/web/apps/docs/content/docs/configuration/intro.mdx @@ -160,6 +160,29 @@ autogenerated from the Rust source — every knob, every default, every constraint. Use this page for understanding, the reference for lookups. +## FAQ + + + + One file per cluster is the simplest way to start, and everything on + this page assumes it. Production deployments usually split into one + config per service — separate CAS, scheduler, and worker files — so + each service can be scaled, restarted, and deployed independently. + + + An in-memory CAS + AC behind one listener — + [`basic_cas.json5`](https://github.com/TraceMachina/nativelink/blob/main/nativelink-config/examples/basic_cas.json5) + in the repo. Fine for a demo, gone on restart; see + [Basic configurations](/configuration/basic) for shapes that survive + a reboot. + + + No — both sections are optional and only required on nodes that + accept `Execute()` calls or run actions. A cache-only node is just + `stores` and `servers`. + + + ## What's next - [Basic configurations](/configuration/basic) — runnable shapes diff --git a/web/apps/docs/content/docs/configuration/production.mdx b/web/apps/docs/content/docs/configuration/production.mdx index f5f7e2f0b..4e81f2b4d 100644 --- a/web/apps/docs/content/docs/configuration/production.mdx +++ b/web/apps/docs/content/docs/configuration/production.mdx @@ -155,6 +155,55 @@ alarm) can read NativeLink's Prometheus output and scale accordingly. The [Kubernetes deployment guide](/deployment/kubernetes) has a working HPA config. +## Filesystem store: page-cache eviction + +The `filesystem` store has an opt-in `evict_page_cache` flag (default +`false`). When enabled, after every blob it writes or reads it asks the +kernel to drop that file's pages from the page cache +(`posix_fadvise(POSIX_FADV_DONTNEED)`). + +Most production deployments should leave it **off**. The common +enterprise topology (an object store with multiple CAS nodes) never +benefits from it, and enabling it carries two costs. + +```json5 +stores: [{ + filesystem: { + content_path: "/var/lib/nativelink/cas", + // Leave this off unless the narrow case below applies. + evict_page_cache: false, + }, +}], +``` + + + On a real filesystem (xfs/ext4), `POSIX_FADV_DONTNEED` takes the + kernel's `lru_add_drain_all()` path: it schedules and waits on a drain + across **every online CPU**, and concurrent callers serialize on a + global lock. The per-call cost therefore grows with the number of + cores, and **adding workers or concurrent actions cannot overcome + it**: every caller converges on the same serialized kernel path, so + throughput is capped no matter how much parallelism you add. On a + many-core host this + turns a busy store into a bottleneck while CPU and disk sit near idle. + (On `tmpfs` the call is a no-op, so this cost is invisible in + tmpfs-backed tests.) + + +The second cost: it evicts the page cache. A store kept on a fast local +disk that relies on the page cache as its hot read tier loses that cache +after every read and write. + +**When to enable it:** only for a filesystem-backed store on a low-core +host where keeping this store's I/O out of the page cache is specifically +desired (for example, to bound page-cache growth on a co-located +filesystem-only deployment). It was originally added to relieve page-cache +pressure that contributed to worker OOM-kills on such a deployment; note +that page cache is reclaimable and the dominant OOM driver is a worker's +anonymous memory (action processes and allocator retention), so this flag +is a narrow mitigation, not a general OOM fix. Do **not** enable it on +many-core hosts or object-store / multi-CAS deployments. + ## TLS and authentication In production, every hop runs over mTLS: @@ -210,6 +259,48 @@ What's worth alerting on: The [Metrics deployment guide](/deployment/metrics) has a Grafana dashboard JSON file you can import. +## Sandboxing + +We've seen in some client configurations issues where Bazel in particular can leave zombie processes around on workers. +To solve this, we've two options around sandboxing that are currently switched off by default for backwards +compatibility, but are recommended for production configurations + +```json5 +workers: [{ + local: { + use_namespaces: true, + use_mount_namespace: true + // rest of your config + } +}] +``` + +`use_namespaces` enables process namespacing for workers, and `use_mount_namespace` then also isolates the worker +root in a new mount namespace. Note, `use_mount_namespace` only works if `use_namespaces` is switched on as well. + +## FAQ + + + + The queue lives in scheduler memory and is restored from the Action + Cache on restart — which is why running two or three stateless + schedulers behind a load balancer is enough for HA. + + + Tag the workers (`gpu: { values: ["nvidia-a100"] }`) and declare the + property as `"exact"` in the scheduler's + `supported_platform_properties`. An action requesting that value is + only ever dispatched to matching workers. + + + No — it never verifies that an object in `fast` also exists in + `slow`. For artifacts that must survive a fast-tier wipe — remote + execution outputs especially — make sure the write path covers both + tiers deliberately. More store gotchas: + [Store overview](/reference/nativelink-config/store-overview#faq). + + + ## A reference production config A working complete production config (CAS + AC + scheduler + workers, diff --git a/web/apps/docs/content/docs/contribute/bazel.mdx b/web/apps/docs/content/docs/contribute/bazel.mdx index f8b9624c2..59c1eb42b 100644 --- a/web/apps/docs/content/docs/contribute/bazel.mdx +++ b/web/apps/docs/content/docs/contribute/bazel.mdx @@ -76,6 +76,21 @@ editor of choice will pick up. reuse Bazel's local cache; pointing at a remote NativeLink cluster makes a fresh clone fast for everyone. -For non-Bazel workflows, see -[Develop with Cargo](/contribute/cargo) or -[Develop with Nix](/contribute/nix). +## FAQ + + + + It bootstraps the toolchain and builds every dependency from source. + Subsequent builds are seconds — and pointing `--remote_cache` at a + NativeLink cluster (above) makes even a fresh clone fast. + + + No — target what you touched + (`bazel test //nativelink-store/tests:s3_store_test`), then run the + full `bazel test //...` once before submitting. + + For non-Bazel workflows, see + [Develop with Cargo](/contribute/cargo) or + [Develop with Nix](/contribute/nix). + + diff --git a/web/apps/docs/content/docs/contribute/cargo.mdx b/web/apps/docs/content/docs/contribute/cargo.mdx index a50a2053c..611846937 100644 --- a/web/apps/docs/content/docs/contribute/cargo.mdx +++ b/web/apps/docs/content/docs/contribute/cargo.mdx @@ -81,3 +81,17 @@ A few flags that help: For the Bazel-based workflow, see [Develop with Bazel](/contribute/bazel). + +## FAQ + + + + Not for most crates — the whole workspace builds with plain Cargo. + You need Bazel for the Bazel-only crates and for a pre-submit + `bazel test //...`, since that's what CI runs. + + + The gRPC services are generated from protobuf definitions at build + time; without `protoc` 3+ on `$PATH`, those generation steps fail. + + diff --git a/web/apps/docs/content/docs/contribute/docs.mdx b/web/apps/docs/content/docs/contribute/docs.mdx index 2f1164a38..c2788831e 100644 --- a/web/apps/docs/content/docs/contribute/docs.mdx +++ b/web/apps/docs/content/docs/contribute/docs.mdx @@ -101,3 +101,20 @@ For tabbed content, use Fumadocs's `Tabs` and `Tab`: Same flow as any other change — see [Contribution guidelines](/contribute/guidelines). Small docs PRs land same-day in most cases. + +## FAQ + + + + At the bottom of the page they belong to — each page ends with a + `## FAQ` section holding the questions readers actually ask about + that material. Add new Q&As there rather than to a standalone FAQ + section; there isn't one. + + + The same pre-commit suite as any PR, including + [Vale](https://vale.sh) style linting and typo checks over the MDX. + Run `pre-commit run --files ` locally to catch + failures before CI does. + + diff --git a/web/apps/docs/content/docs/contribute/guidelines.mdx b/web/apps/docs/content/docs/contribute/guidelines.mdx index fa799c18d..a55605077 100644 --- a/web/apps/docs/content/docs/contribute/guidelines.mdx +++ b/web/apps/docs/content/docs/contribute/guidelines.mdx @@ -63,7 +63,7 @@ We try to make the process predictable. - **First response within 2 business days.** If it's longer, ping us in the PR. -- **Two reviewers for non-trivial changes.** One for typo fixes. +- **One reviewer per change.** Others sometimes chime in. - **Squash on merge.** Every merged PR is one commit on `main`. ## License @@ -79,6 +79,22 @@ Meaningful contributors may be eligible for license waivers for Business Source License modules. Open an issue or contact the maintainers before depending on a waiver. +## FAQ + + + + [Cargo](/contribute/cargo) for small, localized changes — it's the + fastest way in. [Bazel](/contribute/bazel) when you want exactly what + CI runs. [Nix](/contribute/nix) for the fully-pinned shell, and + required for LRE work. + + + CI runs Bazel, and some crates are Bazel-only (the LRE flake outputs, + the Bazel-generated proto bindings). Run `bazel test //...` before + submitting if you touched anything Bazel-specific. + + + ## What's next Pick the dev environment that fits how you work: diff --git a/web/apps/docs/content/docs/contribute/nix.mdx b/web/apps/docs/content/docs/contribute/nix.mdx index db332cb25..3a84fc4d7 100644 --- a/web/apps/docs/content/docs/contribute/nix.mdx +++ b/web/apps/docs/content/docs/contribute/nix.mdx @@ -82,3 +82,19 @@ If you're making a small, localised change — typo fix, doc tweak, a unit test — the Cargo workflow is faster to get into. Nix shines for work touching the build graph, the LRE flow, or anything that needs exact toolchain reproducibility. + +## FAQ + + + + A package manager that builds every package in isolation and + addresses the result by a hash of all its inputs — which is why two + machines running `nix develop` on this repo get bit-identical + toolchains. + + + Only for [LRE](/explanations/lre) work. For everything else it's the + most reproducible option, not the mandatory one — the Cargo and Bazel + workflows stand on their own. + + diff --git a/web/apps/docs/content/docs/deployment/chromium.mdx b/web/apps/docs/content/docs/deployment/chromium.mdx index 30bfd406f..6c1a0a75a 100644 --- a/web/apps/docs/content/docs/deployment/chromium.mdx +++ b/web/apps/docs/content/docs/deployment/chromium.mdx @@ -99,6 +99,20 @@ directory in the source tree has the worker config Samsung uses. toolchain version mismatch between the action that wrote the cache and the one trying to read. +## FAQ + + + + No — pointing Siso at NativeLink is configuration only: GN args plus + the Siso environment variables. The same applies to Chromium forks; + Samsung Internet runs this pattern in production. + + + Yes. A cache-only cluster already cuts incremental build times; add + remote execution later for the full win on link-heavy full builds. + + + ## What's next - [Persistent workers](/deployment/persistent-workers) for the diff --git a/web/apps/docs/content/docs/deployment/kubernetes.mdx b/web/apps/docs/content/docs/deployment/kubernetes.mdx index 7933ab46d..f4b6efc40 100644 --- a/web/apps/docs/content/docs/deployment/kubernetes.mdx +++ b/web/apps/docs/content/docs/deployment/kubernetes.mdx @@ -41,7 +41,7 @@ spec: spec: containers: - name: nativelink - image: ghcr.io/tracemachina/nativelink:v1.4.0 + image: ghcr.io/tracemachina/nativelink:v1.6.1 args: ["/config/cas.json5"] ports: - containerPort: 50051 @@ -104,7 +104,7 @@ spec: spec: containers: - name: nativelink - image: ghcr.io/tracemachina/nativelink:v1.4.0 + image: ghcr.io/tracemachina/nativelink:v1.6.1 args: ["/config/worker.json5"] resources: requests: { cpu: 4, memory: 8Gi } @@ -172,6 +172,26 @@ directory in the source tree has: Clone the repo, point at your cluster, `helm install`. Tune from there. +## FAQ + + + + Yes — one binary can serve CAS, AC, scheduler, and worker from a + single config. Split into the three workloads above once you need to + scale or restart them independently. + + + The CAS owns persistent volumes and a stable identity per replica — + losing one loses cached blobs. Workers hold no state at all; kill and + reschedule them freely, which is exactly what the HPA does. + + + Yes. Every RE-API service is gRPC; an Ingress that downgrades to + plain HTTP/1.1 breaks the protocol. NGINX needs `grpc_pass`; any + modern Gateway API implementation handles it. + + + ## What's next - [Metrics](/deployment/metrics) — wire it into your Grafana. diff --git a/web/apps/docs/content/docs/deployment/metrics.mdx b/web/apps/docs/content/docs/deployment/metrics.mdx index 3b15536f4..8e8b3bdc4 100644 --- a/web/apps/docs/content/docs/deployment/metrics.mdx +++ b/web/apps/docs/content/docs/deployment/metrics.mdx @@ -11,7 +11,7 @@ metrics are available when the store is explicitly wrapped with the opt-in NativeLink metrics are licensed under the Business Source License. Individual developer cache use does not need a commercial license. Teams using metrics in shared, production, or commercial settings -can use NativeLink Cloud, Enterprise, or an intentionally +can use NativeLink Enterprise or an intentionally very inexpensive separate license. See the [license page](https://nativelink.com/license). @@ -168,19 +168,28 @@ results include `hit`, `miss`, `expired`, `success`, and `error`. ### Execution metrics +Base labels below are `execution_instance`, `execution_worker_id` (when a worker +is known), and `execution_priority`. + | Metric | Type | Description | Labels | | ------ | ---- | ----------- | ------ | -| `nativelink_execution_stage_duration` | Histogram | Time spent in each execution stage | `execution_stage` | -| `nativelink_execution_total_duration` | Histogram | Total execution time from submission to completion | `execution_instance` | -| `nativelink_execution_queue_time` | Histogram | Time spent waiting in queue | `execution_priority` | +| `nativelink_execution_stage_duration` | Histogram | Time spent in each worker phase, in seconds | base + `execution_stage` (phase) | +| `nativelink_execution_total_duration` | Histogram | End-to-end time from queued to worker completion, in seconds | base | +| `nativelink_execution_queue_time` | Histogram | Time waiting in queue before a worker started, in seconds | base | | `nativelink_execution_active_count` | Gauge | Current actions in each stage | `execution_stage` | | `nativelink_execution_completed_count_total` | Counter | Completed executions | `execution_result`, `execution_action_digest` | | `nativelink_execution_stage_transitions_total` | Counter | Stage transition events | `execution_instance`, `execution_priority` | -| `nativelink_execution_output_size` | Histogram | Size of execution outputs | - | -| `nativelink_execution_retry_count_total` | Counter | Number of retries | - | +| `nativelink_execution_output_size` | Histogram | Total output bytes (output files plus stdout/stderr) | base | +| `nativelink_execution_retry_count_total` | Counter | Execution retries (a failed attempt that re-queued the action) | base | + +The `execution_stage` label differs by metric. For `nativelink_execution_active_count` +and `nativelink_execution_stage_transitions_total` it is one of `unknown`, +`cache_check`, `queued`, `executing`, or `completed`. For +`nativelink_execution_stage_duration` it is the worker phase: `input_fetch`, +`execution`, or `output_upload`. -Execution stages are `unknown`, `cache_check`, `queued`, `executing`, and -`completed`. +The histograms and `nativelink_execution_retry_count_total` are populated once an +action runs on a worker; cache hits do not emit them. Counter metrics are exposed with a `_total` suffix in Prometheus. The Docker Compose quickstart, recording rules, and included dashboards assume `_total` @@ -270,6 +279,30 @@ storage: out_of_order_time_window: 1h ``` +## FAQ + + + + No — NativeLink emits OTLP only; there is no scrape endpoint. Either + run an OpenTelemetry Collector and let Prometheus scrape the + Collector's exporter, or enable Prometheus's OTLP receiver + (`--web.enable-otlp-receiver`) and send straight to it. The + [quick start](#quick-start) wires up the first path. + + + Metrics are one of the few Business Source License modules. + Individual developer use is fine; shared, production, or commercial + settings need NativeLink Enterprise or an intentionally + inexpensive commercial license — see the + [license page](https://nativelink.com/license). + + + `nativelink_scheduler_queue_depth` — the signal the + [Kubernetes HPA example](/deployment/kubernetes) targets. CPU-based + scaling reacts too slowly because workers are mostly I/O-bound. + + + ## Additional resources - [Metrics deployment examples](https://github.com/TraceMachina/nativelink/tree/main/deployment-examples/metrics) diff --git a/web/apps/docs/content/docs/deployment/oci-object-storage.mdx b/web/apps/docs/content/docs/deployment/oci-object-storage.mdx index 381878bbe..380709535 100644 --- a/web/apps/docs/content/docs/deployment/oci-object-storage.mdx +++ b/web/apps/docs/content/docs/deployment/oci-object-storage.mdx @@ -137,3 +137,17 @@ would for the AWS backend. cache round-trip, a 2 GiB multipart upload (~400 parts) with full byte verification, multipart abort/cleanup, and 200-way concurrent load. It has not been benchmarked at sustained production scale or duration. + +## FAQ + + + + Yes — segment them with `key_prefix` (`cas/` and `ac/` in the example + above) rather than provisioning two buckets. + + + `aws`, `gcs`, `azure`, `ontap`, and `r2`, alongside `oci` — one store + type, six providers, sharing the same multipart, retry, and streaming + code paths. + + diff --git a/web/apps/docs/content/docs/deployment/on-prem-overview.mdx b/web/apps/docs/content/docs/deployment/on-prem-overview.mdx index 274c0e45b..d035b630d 100644 --- a/web/apps/docs/content/docs/deployment/on-prem-overview.mdx +++ b/web/apps/docs/content/docs/deployment/on-prem-overview.mdx @@ -107,7 +107,7 @@ contracts; truly global teams. Numbers we've seen in production. Yours will vary — these are the order-of-magnitude starting points: -| Resource | Per active developer / week | +| Resource | Per active developer | | --------------------- | ------------------------------- | | CAS storage (C++) | 15-20 GB | | CAS storage (Go/Rust) | 5-10 GB | @@ -118,6 +118,29 @@ order-of-magnitude starting points: When the cluster is healthy, the scheduler is the cheapest piece and CAS storage is the dominant cost. Plan accordingly. +## FAQ + + + + The CAS. Schedulers and workers are stateless, and the Action Cache + can be wiped harmlessly — losing it just forces re-execution until it + warms back up. That asymmetry drives backup policy: snapshot the CAS + backend, treat everything else as disposable. + + + On scheduler queue depth, not CPU — workers spend most of their time + blocked on I/O, so CPU-based scaling reacts too late. The + [Kubernetes guide](/deployment/kubernetes) has a working HPA + targeting `nativelink_scheduler_queue_depth`. + + + Filesystem CAS: `rsync` or ZFS/Btrfs snapshots. S3-compatible: + versioning, lifecycle policies, and cross-region replication for DR. + Redis: ephemeral by design — loss is a cache miss. The procedure is + in [NativeLink on-prem](/getting-started/on-prem). + + + ## What's next - [Kubernetes](/deployment/kubernetes) — working Helm chart. diff --git a/web/apps/docs/content/docs/deployment/persistent-workers.mdx b/web/apps/docs/content/docs/deployment/persistent-workers.mdx index 9ebdbddd0..a77c84043 100644 --- a/web/apps/docs/content/docs/deployment/persistent-workers.mdx +++ b/web/apps/docs/content/docs/deployment/persistent-workers.mdx @@ -15,7 +15,7 @@ cost is paid once; subsequent actions reuse the same process. Remote persistent workers are licensed under the Business Source License. Individual developer cache use does not need a commercial license. Teams using persistent workers in shared, production, or -commercial settings can use NativeLink Cloud, Enterprise, or an +commercial settings can use NativeLink Enterprise or an intentionally very inexpensive separate license. See the [license page](https://nativelink.com/license). @@ -110,6 +110,22 @@ clusters don't waste memory. a soft mitigation; for serious workloads, set a hard `max_invocations` and recycle. +## FAQ + + + + Rarely — plain `clang` starts in ~10 ms, so the win is negligible and + not worth the caveats. They pay back almost immediately on JVM-family + toolchains (`javac`, `kotlinc`) and modestly on `rustc`. + + + They can — a warm process isn't sandboxed by default, so a toolchain + that mutates global state between invocations produces + nondeterministic results. Sandbox the worker process itself and + recycle workers periodically (see [Caveats](#caveats)). + + + ## What's next - [Configuration → Production](/configuration/production) — the diff --git a/web/apps/docs/content/docs/explanations/architecture.mdx b/web/apps/docs/content/docs/explanations/architecture.mdx index eae6bbe12..e416e061e 100644 --- a/web/apps/docs/content/docs/explanations/architecture.mdx +++ b/web/apps/docs/content/docs/explanations/architecture.mdx @@ -140,6 +140,28 @@ Three operational properties drove the design: The same simplicity at the protocol level is why NativeLink can sustain over a billion build requests a month on modest hardware. +## FAQ + + + + Only the CAS — losing it loses every cached artifact. The AC is a + lookup table you can lose harmlessly: clients re-execute and refill + it. Schedulers and workers hold no durable state at all. + + + Yes — the same executable serves any subset, chosen by config. + Single-developer setups run everything in one process; teams split + the services so each scales independently. + + + A namespace within one cluster: action hashes from different instance + names never collide, so one deployment can serve isolated teams. + Every client and server must agree on it — a mismatch looks like a + permanently empty cache. More vocabulary in the + [glossary](/reference/glossary). + + + ## What's next - [Local Remote Execution](/explanations/lre) — running the same diff --git a/web/apps/docs/content/docs/explanations/history.mdx b/web/apps/docs/content/docs/explanations/history.mdx index 3631ac4d2..8659694fc 100644 --- a/web/apps/docs/content/docs/explanations/history.mdx +++ b/web/apps/docs/content/docs/explanations/history.mdx @@ -70,3 +70,22 @@ working examples, a clear path from `setup` to `production`. If you find a gap, the docs source lives in [`web/apps/docs/content/docs`](https://github.com/TraceMachina/nativelink) and PRs are welcome. + +## FAQ + + + + Three operational reasons: no GC pauses (tail latency comes from + network and storage, not the runtime), memory safety (a scheduler + memory bug could silently serve a wrong artifact org-wide — Rust + removes the class at compile time), and throughput (a single instance + sustains over a billion requests a month with sub-millisecond `p99` + lookups). + + + Compile times — mitigated by sccache backed by NativeLink itself — + and a smaller hiring pool, which has hurt less than expected: the + people who write distributed-systems Rust tend to be exactly the + people you want writing distributed systems. + + diff --git a/web/apps/docs/content/docs/explanations/lre.mdx b/web/apps/docs/content/docs/explanations/lre.mdx index ff7ad1d21..23f7b5761 100644 --- a/web/apps/docs/content/docs/explanations/lre.mdx +++ b/web/apps/docs/content/docs/explanations/lre.mdx @@ -92,9 +92,8 @@ The recommended flow: build --remote_executor=grpcs://TODO ``` - Fill these in with either your [dev.nativelink.com](https://dev.nativelink.com) - credentials or a self-hosted cluster with a worker capable of the - platform the example needs. A plain + Fill these in with a self-hosted cluster that has a worker capable + of the platform the example needs. A plain [local instance](/rbe/local-testing) (drop the `s` in `grpcs://`, point at `127.0.0.1`) is enough to validate `remote_cache`, but the C++ example's `lre-cc` platform still needs a real worker @@ -135,8 +134,35 @@ filesystem state. That's the entire reason. Nix gives us hash-pinned tools; everything else follows. -For background, see [What is Nix?](/faq/nix) and -[How do I make my Bazel setup hermetic?](/faq/hermeticity). +For background on Nix itself, see +[Develop with Nix](/contribute/nix); for the hermeticity checklist, +see the [FAQ](#faq) below. + +## FAQ + + + + Pin the toolchain (Nix or `rules_cc`'s toolchain abstraction — + never `/usr/bin/cc`), pin every `http_archive` URL and sha256, + add `build --incompatible_strict_action_env` to stop `$PATH` + leaks, enable strict include checking + (`--features=layering_check`), and compare action hashes across + machines with `bazel aquery`. LRE gives you the toolchain-pinning + piece for free. + + + Build the same target twice on the same machine under LRE and + compare outputs — anything that differs is a leak. At the cluster + level, non-hermetic actions surface as cache misses where you + expected hits, or as nondeterministic action results. + + + No — it solves determinism, not throughput. Fan-out across + hundreds of cores, or building for platforms you can't host, + still wants remote workers. The point is that action hashes + match, so local and remote execution share one cache. + + ## What's next diff --git a/web/apps/docs/content/docs/faq/caching.mdx b/web/apps/docs/content/docs/faq/caching.mdx deleted file mode 100644 index c22a9772c..000000000 --- a/web/apps/docs/content/docs/faq/caching.mdx +++ /dev/null @@ -1,53 +0,0 @@ ---- -title: What is remote caching? -description: Reuse build outputs across machines so you only pay for compilation once. ---- - -Remote caching is the practice of storing the result of every build -action in a shared store, keyed by the hash of the action's inputs. If -anyone has already compiled the file you're about to compile — a -teammate, your CI, an agent — you fetch the result instead of doing -the work. - -## How the hash gets computed - -Take everything that affects a compilation's output: - -- The source files (each by content hash). -- The compiler binary (by content hash). -- The compile command, exactly as it'll be executed. -- The set of header files included transitively. -- Anything else the build system declared as an input. - -Hash that whole bundle. Two invocations with the same bundle produce -identical outputs; the cache returns the prior result. One byte -different anywhere — a header you didn't realise you depended on, a -patch version bump in the compiler — and the hash changes, the cache -misses, the work runs fresh. - -This only works if the build is **hermetic** — i.e. every input is -declared. See [hermeticity](/faq/hermeticity). - -## What a cache hit looks like - -A typical Bazel hit: - -``` -INFO: From Compiling src/foo.cc: -(remote cache hit) -Target //src:foo up-to-date: - bazel-bin/src/libfoo.a -``` - -The action ran in 2-10 ms. The compiler never started; the linker -never touched it. You paid the cost of one HTTP request. - -## Where NativeLink fits - -NativeLink implements the standard -[Remote Execution API](https://github.com/bazelbuild/remote-apis) — -the same protocol Bazel, Buck2, Siso, Goma, and Pants speak. You -point your build at a NativeLink server, and every action gets cached -automatically. - -The fast version: [Setup](/getting-started/setup). diff --git a/web/apps/docs/content/docs/faq/cost.mdx b/web/apps/docs/content/docs/faq/cost.mdx deleted file mode 100644 index b4a594b97..000000000 --- a/web/apps/docs/content/docs/faq/cost.mdx +++ /dev/null @@ -1,60 +0,0 @@ ---- -title: Is NativeLink free? -description: Yes — NativeLink is source-available and free for permitted self-hosted use. Cloud and Enterprise are paid offerings. ---- - -**Yes — NativeLink is source-available and free for permitted -self-hosted use.** Most of the monorepo is licensed under -`FSL-1.1-Apache-2.0`, which allows internal use, modification, and -redistribution for non-competing purposes, then grants an Apache 2.0 -future license on the schedule described in the repository -[`LICENSE`](https://github.com/TraceMachina/nativelink/blob/main/LICENSE). -The customer-facing summary lives on the -[NativeLink license page](https://nativelink.com/license). - -NativeLink is not a single-license monorepo. Some feature modules -carry their own source headers. In particular, metrics and remote -persistent workers are licensed under the Business Source License. -Developers using NativeLink for an individual cache do not need a -commercial license. Teams using those modules in shared, production, -or commercial settings can use NativeLink Cloud, Enterprise, or a -separate commercial license, which is intentionally very inexpensive. -Meaningful contributors may be eligible for license waivers; contact -the maintainers before relying on one. - -If you'd rather pay someone to run it for you, two paid tiers exist: - -- **Cloud** — managed multi-tenant cluster, flat $999+/month. Full - SLA, autoscaling, dashboard. -- **Enterprise** — single-tenant deployment, dedicated solutions - engineer, on-prem or managed. Custom contracts. - -The -[pricing page](https://nativelink.com/pricing) has the feature -comparison. - -## What does "free" actually cover - -- The full RE-API server: CAS, AC, scheduler, worker. -- Every storage backend: filesystem, S3, Redis, GCS, Azure Blob. -- Every supported build client: Bazel, Buck2, Siso, Pants, Goma. -- The CLI tooling and Helm chart. - -No artificial limits on cache size, action count, worker count, or -team size. The self-hosted build is the same codebase we run in -production, subject to the license that applies to each module, minus -the operational tooling and dedicated support that Cloud and -Enterprise add. - -## What you'd pay for instead - -Hosting NativeLink yourself costs: - -- Compute for the control plane (typically a few small VMs). -- Storage for the CAS (S3 / equivalent, sized for ~5-20 GB per active - developer per week). -- Worker compute (variable; this is the part Cloud rents to you). -- Engineer hours running it. - -For most teams under 30 engineers, self-host is cheaper. Past that, -Cloud usually wins on engineer-time alone. diff --git a/web/apps/docs/content/docs/faq/hermeticity.mdx b/web/apps/docs/content/docs/faq/hermeticity.mdx deleted file mode 100644 index e9ae2844a..000000000 --- a/web/apps/docs/content/docs/faq/hermeticity.mdx +++ /dev/null @@ -1,68 +0,0 @@ ---- -title: How do I make my Bazel setup hermetic? -description: Eliminate every "it works on my machine" by declaring every input. ---- - -A hermetic build is one where every input is declared, every output -is reproducible, and the result depends only on the declared inputs. -Same recipe, same ingredients, same dish, every time. - -Bazel **can** produce hermetic builds. Out of the box, it doesn't — -most Bazel projects depend implicitly on system tooling, environment -variables, and the host OS. Reaching real hermeticity takes -deliberate work. - -## The hermeticity checklist - - -
  • - **Pin the toolchain.** Don't rely on `/usr/bin/cc`. Use - `rules_cc`'s toolchain abstraction or a Nix-provided toolchain; - register the platform in `MODULE.bazel`. -
    • -
  • - **Lock dependency sources.** Every `http_archive` should pin both - the URL and the sha256. Module dependencies should pin a version, - not a range. -
    • -
  • - **Forbid `$PATH` leaks.** Add - `build --incompatible_strict_action_env` to your `.bazelrc`. - Bazel will now only forward an explicitly-allowed set of env - vars. -
    • -
  • - **Use the sandbox you trust.** On Linux: `linux-sandbox`. On - macOS: `darwin-sandbox`. For maximum isolation, run actions in a - container or under [LRE](/explanations/lre). -
    • -
  • - **Declare every header.** Strict include-checking - (`--features=layering_check`) catches the headers your `cc_library` - silently transitively depends on. -
    • -
  • - **Compare action hashes across machines.** If the same target - produces different hashes on two laptops, something is leaking. - `bazel aquery 'mnemonic("CppCompile", //target:label)' --output=text` - shows you the action's full input set. -
    • -     - -## How NativeLink helps - -A hermetic build maps directly to a content-addressed cache: same -inputs = same hash = same cached result. NativeLink doesn't *make* -your build hermetic, but it surfaces failures fast — non-hermetic -actions show up as cache misses where there should be hits, or, worse, -as nondeterministic action results. - -The fastest way to find leaks: enable -[LRE](/explanations/lre), build the same target twice on the -same machine, and compare outputs. Anything that differs is a leak. - -## Further reading - -- [What are toolchains?](/faq/toolchains) -- [What is Nix?](/faq/nix) -- [LRE](/explanations/lre) — the easiest way to get there. diff --git a/web/apps/docs/content/docs/faq/lre.mdx b/web/apps/docs/content/docs/faq/lre.mdx deleted file mode 100644 index d781c04d5..000000000 --- a/web/apps/docs/content/docs/faq/lre.mdx +++ /dev/null @@ -1,48 +0,0 @@ ---- -title: What is LRE? -description: Local Remote Execution — the hermeticity of RBE without the network. ---- - -LRE (Local Remote Execution) runs a NativeLink worker on your laptop -and routes every build action through it. You get the hermeticity -guarantees of remote execution — content-addressed inputs, pinned -toolchain, sandboxed processes — without sending anything over the -network. - -## What it solves - -The classic developer-experience trade-off: - -- **Pure local build** — fast feedback, "works on my machine" bugs. -- **Pure remote build** — bit-identical with CI, slow on every change. - -LRE is the third option: bit-identical with CI, fast on every change. -Cache hits from the remote cluster apply locally; cache writes from -local apply remotely. - -## How it works - -Three pieces: - -1. A NativeLink server bound to `localhost`. -2. A Nix-pinned toolchain: every binary the build invokes has a - content-addressed path in the Nix store. -3. A configured build client (Bazel, Buck2, ...) targeting the local - server. - -The build issues the same `Execute` RPCs it would issue against a -remote cluster — they just happen to land on `localhost`. - -## When to use LRE - -| Scenario | LRE | -| ---------------------------------------------- | ---- | -| Solo dev iterating, want CI reproducibility | ✓ | -| Team sharing a cache without a server | ✓ | -| Repro a CI failure locally | ✓ | -| Fan-out across 200 cores | ✗ | -| Building for a platform you can't host | ✗ | - -The split: LRE for determinism, remote workers for throughput. - -The full write-up: [Explanations → LRE](/explanations/lre). diff --git a/web/apps/docs/content/docs/faq/meta.json b/web/apps/docs/content/docs/faq/meta.json deleted file mode 100644 index 22e2d0e38..000000000 --- a/web/apps/docs/content/docs/faq/meta.json +++ /dev/null @@ -1,13 +0,0 @@ -{ - "pages": [ - "cost", - "caching", - "remote-execution", - "lre", - "toolchains", - "hermeticity", - "nix", - "rust" - ], - "title": "FAQ" -} diff --git a/web/apps/docs/content/docs/faq/nix.mdx b/web/apps/docs/content/docs/faq/nix.mdx deleted file mode 100644 index 07c0b07ee..000000000 --- a/web/apps/docs/content/docs/faq/nix.mdx +++ /dev/null @@ -1,49 +0,0 @@ ---- -title: What is Nix? -description: A purely functional package manager that gives NativeLink its hermeticity guarantees. ---- - -Nix is a package manager that builds every package in isolation, with -a hash that encodes every input — source code, dependencies, build -flags, even the version of `gcc` used to compile the dependencies. -Two machines running the same Nix flake produce bit-identical -artifacts. - -That property — hash-pinned, content-addressed everything — is what -NativeLink needs for [LRE](/explanations/lre) and is why we use -Nix to provision toolchains. - -## What it gives you, concretely - -- **Toolchains in the Nix store.** Your `clang` lives at - `/nix/store/abc123-clang-18.1.6/bin/clang`. The path itself is the - hash. NativeLink can use that path as an input identifier directly. -- **No "global" state to leak.** A Nix shell exposes only the - packages you ask for. There's no `/usr/lib` to accidentally - depend on. -- **Reproducible across machines.** `nix develop` on your laptop and - on CI produces the same shell, byte-for-byte. - -## What you give up - -- A learning curve. The Nix language is unfamiliar; the error - messages take getting used to. -- Initial download time. The first `nix develop` for a big project - fetches gigabytes. - -## How to start - -The -[next-gen Nix installer](https://github.com/NixOS/experimental-nix-installer) -is the smoothest path. After installing: - -```bash -git clone https://github.com/TraceMachina/nativelink -cd nativelink -nix develop -``` - -You're now in a shell with the exact toolchain CI uses — Rust, Bazel, -mold, `protoc`, the works. - -For a full LRE setup, see [Explanations → LRE](/explanations/lre). diff --git a/web/apps/docs/content/docs/faq/remote-execution.mdx b/web/apps/docs/content/docs/faq/remote-execution.mdx deleted file mode 100644 index 2cf130d5b..000000000 --- a/web/apps/docs/content/docs/faq/remote-execution.mdx +++ /dev/null @@ -1,62 +0,0 @@ ---- -title: What is remote execution? -description: Build steps run on a worker fleet, not your laptop. Faster builds, lower bills. ---- - -Remote execution ships the build action — inputs, command, sandbox -spec — to a fleet of workers and brings back the outputs. Your laptop -or CI runner becomes a thin coordinator; the heavy lifting happens on -hardware sized for it. - -## The shape - -``` -[ client ] ─ Execute() ─▶ [ scheduler ] - │ - ┌───────┼───────┐ - ▼ ▼ ▼ - [worker] [worker] [worker] - │ │ │ - └───────┴───────┘ - ▼ - [ CAS / AC ] -``` - -The client tells the scheduler "run this action." The scheduler picks -a worker that satisfies the action's platform requirements. The -worker fetches inputs from the content-addressed store, runs the -command in a sandbox, uploads outputs back to the store, and reports -the result. - -The cache from [remote caching](/faq/caching) sits underneath -all of this — actions hit it first; only misses get executed. - -## Why ship the work elsewhere - -Three reasons, in roughly the order teams adopt them: - -1. **Throughput.** A worker fleet has more cores than any one - developer machine. Linker steps that take 60 seconds on a laptop - take 8 on a hot worker. -2. **Hermeticity.** The worker has exactly one job, in exactly one - environment. No conflicting versions of system libraries, no - stale `/tmp`, no surprises. -3. **Cost.** CI runners are expensive because they sit idle most of - the day. A worker fleet utilises hardware constantly — average - utilisation goes from 5-10% to 60-80%. - -## Where NativeLink fits - -NativeLink is a Remote Execution API server. Any build tool that -speaks the protocol — -[Bazel](https://bazel.build), -[Buck2](https://buck2.build), -[Siso](https://chromium.googlesource.com/build/+/refs/heads/main/siso/README.md), -[Pants](https://www.pantsbuild.org), -[Goma](https://chromium.googlesource.com/infra/goma/client/) — -plugs in with no code changes. - -The flow looks like the diagram above, with NativeLink as the -scheduler + worker fleet. See -[Architecture](/explanations/architecture) for the full -walkthrough. diff --git a/web/apps/docs/content/docs/faq/rust.mdx b/web/apps/docs/content/docs/faq/rust.mdx deleted file mode 100644 index d68b53da1..000000000 --- a/web/apps/docs/content/docs/faq/rust.mdx +++ /dev/null @@ -1,44 +0,0 @@ ---- -title: Why Rust? -description: Memory safety without garbage collection — the right shape for a service that has to be fast and never lie. ---- - -Three reasons: - -## No GC pauses - -Build infrastructure runs under sustained load. A scheduler hiccup -during a `p99` request is the difference between "the build felt -fast" and "the build was unpredictable." Garbage-collected runtimes -insert that hiccup at exactly the wrong time, sized roughly to the -heap. - -Rust has no garbage collector. The tail latency on a NativeLink -cluster comes from network and storage, not the language runtime. - -## Memory safety - -A miscompiled binary that ships across an organisation is a serious -problem. Worse, build systems are pure-function critical -infrastructure: a memory bug in the scheduler could silently produce -the wrong artifact and serve it from cache to every developer. - -Rust catches those bugs at compile time. The class of incident -involving "we shipped something that wasn't what we compiled" doesn't -happen. - -## Throughput - -A single NativeLink instance handles over a billion build requests -per month on modest hardware. That's `~400 req/s` sustained, -`p99 < 1ms` lookups. Hitting those numbers in a GC'd language -requires more horizontal scaling, more memory, more tuning. - -## What it costs - -- Compile times. Cargo builds are slower than `go build`. Mitigated - by sccache + NativeLink itself. -- Smaller talent pool. Hiring Rust engineers is harder than hiring Go - engineers. We've found this less painful than expected — the people - who write distributed-system Rust tend to be exactly the people you - want writing distributed systems. diff --git a/web/apps/docs/content/docs/faq/toolchains.mdx b/web/apps/docs/content/docs/faq/toolchains.mdx deleted file mode 100644 index 248c38aa9..000000000 --- a/web/apps/docs/content/docs/faq/toolchains.mdx +++ /dev/null @@ -1,52 +0,0 @@ ---- -title: What are toolchains? -description: How NativeLink represents compilers, linkers, and the runtime they need. ---- - -A toolchain is the bundle of binaries and configuration NativeLink -needs to run a build action — the compiler, the linker, the standard -library headers, the `cc` wrapper script, any shared library the -compiler depends on. Everything that touches the action. - -For caching and hermeticity to work, the toolchain has to be **fully -declared as inputs**. If the build implicitly picks up `clang` from -`$PATH`, two machines with slightly different `clang` versions produce -different outputs but identical action hashes. The cache lies; you -debug for an afternoon. - -## Three ways to provide one - -| Provider | Hermetic? | Setup cost | -| ------------------------- | ------------ | -------------- | -| **Nix flake** | Fully | Highest | -| **Bazel rules_cc / rules_rust** | Mostly | Medium | -| **System (`$PATH`)** | No | Lowest | - -### Nix - -The recommended path for [LRE](/explanations/lre) and any -deployment where reproducibility matters. The toolchain is identified -by a hash of its inputs; two machines pulling the same flake get -byte-identical binaries. - -### Bazel rules_cc / rules_rust - -The Bazel-native option. Pin the toolchain version in your -`MODULE.bazel`, register it as a platform, point actions at it. -Reasonable for most Bazel monorepos. - -### System tooling - -Use whatever's installed. Quick to set up, no hermeticity guarantees. -Reasonable for "I just want to try the cache out for a day." - -## Tagging actions with a toolchain - -NativeLink matches actions to workers via `platform_properties`. A -worker that has the `nvidia-a100` GPU tagged will be the only worker -that receives actions requesting that GPU. Same applies to toolchain -versions, OS, CPU count, custom labels — anything the cluster -operator configures. - -For the configuration knobs see -[Configuration → Production](/configuration/production#scheduler-ha-with-sticky-workers). diff --git a/web/apps/docs/content/docs/getting-started/on-prem.mdx b/web/apps/docs/content/docs/getting-started/on-prem.mdx index fb47d20ae..c7834b1c1 100644 --- a/web/apps/docs/content/docs/getting-started/on-prem.mdx +++ b/web/apps/docs/content/docs/getting-started/on-prem.mdx @@ -13,18 +13,19 @@ serves your team without 3 AM pages. Pick on-prem when one of these is true: - **Data residency.** Build artifacts can carry source code, - toolchains, even credentials. If those have to stay in a specific - region (EU GDPR, US FedRAMP boundaries, air-gapped corp networks), - managed Cloud isn't an option yet. + toolchains, even credentials. Self-hosting keeps them in whatever + region or boundary you need (EU GDPR, US FedRAMP boundaries, + air-gapped corp networks). - **Specialised hardware.** GPU workers, ARM cross-compile fleets, - in-house silicon. Cloud supports the common cases; on-prem lets you - use anything that can run a Linux binary. + in-house silicon. On-prem lets you use anything that can run a + Linux binary. - **You already operate stateful services.** If your team owns Kubernetes, Postgres, S3-compatible storage, adding NativeLink is marginal work. -If none of those apply, NativeLink Cloud is cheaper, faster to -provision, and one fewer pager rotation. +If you'd rather not run it yourself, +[Enterprise](https://nativelink.com/pricing) offers a managed +deployment. ## What ships in the box @@ -66,7 +67,7 @@ each independently.
  • **Plan capacity.** Heuristics from production clusters: - - CAS storage: 5–20 GB per active developer per week, depending on + - CAS storage: 5–20 GB per active developer, depending on language. C++ skews high; Go skews low. - Worker CPU: 1 vCPU per concurrent action. Headroom matters more than peak. @@ -92,7 +93,7 @@ specific version your config references — `latest` works but pinning avoids surprises. ```bash -docker pull ghcr.io/tracemachina/nativelink:v1.4.0 +docker pull ghcr.io/tracemachina/nativelink:v1.6.1 ``` The [`/pkgs/container/nativelink`](https://github.com/TraceMachina/nativelink/pkgs/container/nativelink) @@ -113,6 +114,37 @@ clients. Snapshot strategy depends on the backend: The Action Cache can be wiped without data loss; you'll re-execute everything until it warms back up. +## FAQ + + + + Yes, for permitted use. Most of the monorepo is + `FSL-1.1-Apache-2.0` — internal use, modification, and redistribution + for non-competing purposes are allowed. Metrics and remote persistent + workers are Business Source License modules: fine for an individual + developer's cache, but shared, production, or commercial use of those + modules needs NativeLink Enterprise or an intentionally + inexpensive commercial license. Meaningful contributors may be + eligible for waivers — contact the maintainers before relying on one. + The practical summary is on the + [license page](https://nativelink.com/license). + + + Compute for the control plane (a few small VMs), CAS storage — the + dominant cost in a healthy cluster — worker compute, and the engineer + hours to run it all. For most teams under ~30 engineers, self-hosting + is cheaper; past that, a managed + [Enterprise](https://nativelink.com/pricing) deployment + usually wins on engineer time alone. + + + No. Nothing artificially limits cache size, action count, + worker count, or team size — the self-hosted build is the same + codebase run in production, subject to each module's license, minus + the operational tooling and support the paid tiers add. + + + ## What's next - [Configuration → Production](/configuration/production) — the diff --git a/web/apps/docs/content/docs/getting-started/other-build-systems/buck2.mdx b/web/apps/docs/content/docs/getting-started/other-build-systems/buck2.mdx index f734543ca..d70d95914 100644 --- a/web/apps/docs/content/docs/getting-started/other-build-systems/buck2.mdx +++ b/web/apps/docs/content/docs/getting-started/other-build-systems/buck2.mdx @@ -91,3 +91,19 @@ download paths are working. sets `remote_enabled = True`. - **Instance-name errors.** Make sure Buck2's `instance_name` matches the NativeLink config. The integration test uses `main`. + +## FAQ + + + + `engine_address`, `action_cache_address`, and `cas_address` are + separate settings because Buck2 lets you split those services across + hosts. NativeLink serves them all from one listener, so all three + point at the same `host:port`. + + + Both. Execution needs an execution platform registered with + `remote_enabled = True` (above) plus a NativeLink scheduler and + worker — the integration test config runs one of each. + + diff --git a/web/apps/docs/content/docs/getting-started/other-build-systems/buildstream.mdx b/web/apps/docs/content/docs/getting-started/other-build-systems/buildstream.mdx index da02be24c..556f39513 100644 --- a/web/apps/docs/content/docs/getting-started/other-build-systems/buildstream.mdx +++ b/web/apps/docs/content/docs/getting-started/other-build-systems/buildstream.mdx @@ -115,3 +115,19 @@ and checks that NativeLink did not log an error. - **Remote actions do not match a worker.** Compare the platform properties BuildStream requests with the properties your NativeLink worker advertises. The integration test worker advertises `ISA: x86-64`. + +## FAQ + + + + They configure different BuildStream subsystems — artifact push/pull + storage versus the execution, action-cache, and storage services used + for remote builds. Pointing every URL at the same NativeLink listener + is fine; it exposes all of those services on one port. + + + BuildStream expresses these endpoints as URLs rather than bare + `host:port` pairs. The checked-in integration test runs without TLS, + hence `http://localhost:50051`. + + diff --git a/web/apps/docs/content/docs/getting-started/other-build-systems/cmake-recc.mdx b/web/apps/docs/content/docs/getting-started/other-build-systems/cmake-recc.mdx index a0b8731c7..b2167a1f3 100644 --- a/web/apps/docs/content/docs/getting-started/other-build-systems/cmake-recc.mdx +++ b/web/apps/docs/content/docs/getting-started/other-build-systems/cmake-recc.mdx @@ -36,19 +36,16 @@ giving you cross-machine and cross-branch caching. ## 1. Start NativeLink ```sh -curl -O https://raw.githubusercontent.com/TraceMachina/nativelink/v1.0.0/nativelink-config/examples/basic_cas.json5 +curl -O https://raw.githubusercontent.com/TraceMachina/nativelink/v1.6.1/nativelink-config/examples/basic_cas.json5 docker run -d --name nativelink \ - --platform linux/amd64 \ -v $(pwd)/basic_cas.json5:/config \ -p 50051:50051 -p 50061:50061 \ - ghcr.io/tracemachina/nativelink:v1.0.0 config + ghcr.io/tracemachina/nativelink:v1.6.1 config ``` -The prebuilt image is x86_64. `--platform linux/amd64` is a no-op on Linux -x86_64 and triggers fast emulation on Apple Silicon. (Linux arm64 users: -build NativeLink with `nix run github:TraceMachina/nativelink ./basic_cas.json5` -instead.) +The image is multi-arch (x86_64 and ARM64) as of v1.6.0, so it runs +natively on Linux and Apple Silicon alike. Confirm it came up: @@ -195,6 +192,27 @@ actions, not link actions, by default. docker stop nativelink && docker rm nativelink ``` +## FAQ + + + + Not in this setup — `RECC_CACHE_ONLY=1` means misses compile locally + and only the outputs travel through NativeLink. Drop that variable + once you have remote workers configured and recc will dispatch + compiles to them. + + + recc ships compile actions only by default; links always run locally. + For remote link steps you want a full RBE client — see + [Classic RBE examples](/rbe/examples). + + + Point `RECC_SERVER` at a shared NativeLink deployment and keep + `RECC_INSTANCE` identical across machines — mismatched instance names + produce a silently empty cache. + + + ## Going further - Point `RECC_SERVER` at a shared NativeLink deployment (cloud or diff --git a/web/apps/docs/content/docs/getting-started/other-build-systems/index.mdx b/web/apps/docs/content/docs/getting-started/other-build-systems/index.mdx index f550eb3b5..dbd1652e0 100644 --- a/web/apps/docs/content/docs/getting-started/other-build-systems/index.mdx +++ b/web/apps/docs/content/docs/getting-started/other-build-systems/index.mdx @@ -35,3 +35,27 @@ For a local Docker server, start with [Getting Started → Setup](/getting-started/setup). For the Buck2 integration test shape, see [`integration_tests/buck2/buck2_cas.json5`](https://github.com/TraceMachina/nativelink/tree/main/integration_tests/buck2/buck2_cas.json5). + +## FAQ + + + + Yes — they speak the same protocol against the same endpoint. Each + tool hashes its actions differently, so tools won't share entries + with each other, but every client of the same tool and toolchain + will. + + + Action hashes computed under different `instance_name` values never + collide, and a mismatch produces no error — just a cache that looks + permanently empty. Every client targeting a cache must send the same + instance name the server exposes. + + + If you're on CMake, + [recc in cache-only mode](/getting-started/other-build-systems/cmake-recc) + needs no worker fleet and no build-file changes. For Bazel, two + `.bazelrc` lines against the [Setup](/getting-started/setup) server + do it. + + diff --git a/web/apps/docs/content/docs/getting-started/other-build-systems/pants.mdx b/web/apps/docs/content/docs/getting-started/other-build-systems/pants.mdx index 033ae8e69..e1da90fb0 100644 --- a/web/apps/docs/content/docs/getting-started/other-build-systems/pants.mdx +++ b/web/apps/docs/content/docs/getting-started/other-build-systems/pants.mdx @@ -55,3 +55,19 @@ and toolchains aligned. config. The local examples use `main`. - **Connection failures.** Pants expects a `grpc://` URL in `remote_store_address`. + +## FAQ + + + + To let you split roles: CI can populate the cache + (`remote_cache_write = true`) while developer laptops only read from + it. For a single shared setup, enable both. + + + Yes — it's the same Remote Execution API — but the worker environment + must match what Pants sends. Get remote caching working first, then + align worker platform properties and toolchains before enabling + execution. + + diff --git a/web/apps/docs/content/docs/getting-started/other-build-systems/siso.mdx b/web/apps/docs/content/docs/getting-started/other-build-systems/siso.mdx index 2fa2f38ab..93b08200e 100644 --- a/web/apps/docs/content/docs/getting-started/other-build-systems/siso.mdx +++ b/web/apps/docs/content/docs/getting-started/other-build-systems/siso.mdx @@ -66,3 +66,18 @@ On a warm cache, cache hits should climb and network errors should stay at zero. If actions fall back to local execution, compare the platform properties Siso requests with the properties your NativeLink workers advertise. + +## FAQ + + + + Only for TLS-free local development. Against any shared endpoint, use + the TLS variables shown above instead. + + + Siso replaces Ninja and speaks full remote execution — it's the path + for Chromium-style builds. recc is a compiler launcher for ordinary + CMake projects that want caching with minimal setup — see + [CMake with recc](/getting-started/other-build-systems/cmake-recc). + + diff --git a/web/apps/docs/content/docs/getting-started/setup.mdx b/web/apps/docs/content/docs/getting-started/setup.mdx index dc5250d9e..66c0f6b0f 100644 --- a/web/apps/docs/content/docs/getting-started/setup.mdx +++ b/web/apps/docs/content/docs/getting-started/setup.mdx @@ -18,22 +18,24 @@ system to the running cluster. The fastest way is the prebuilt container image. It runs anywhere Docker runs and ships with a tested configuration. - + - + ```bash # Grab a known-good basic configuration -curl -O https://raw.githubusercontent.com/TraceMachina/nativelink/v1.4.0/nativelink-config/examples/basic_cas.json5 +curl -O https://raw.githubusercontent.com/TraceMachina/nativelink/v1.6.1/nativelink-config/examples/basic_cas.json5 # Run the official image docker run \ -v $(pwd)/basic_cas.json5:/config \ -p 50051:50051 \ - ghcr.io/tracemachina/nativelink:v1.4.0 config + ghcr.io/tracemachina/nativelink:v1.6.1 config ``` -The server is now listening on `localhost:50051`. +The image is multi-arch — x86_64 and ARM64 both run natively (use +v1.6.0 or later; earlier tags are x86_64-only). The server is now +listening on `localhost:50051`. @@ -46,8 +48,9 @@ nix run github:TraceMachina/nativelink ./basic_cas.json5 ``` Slower than the prebuilt image because it builds from source, but works on -macOS (Apple Silicon and Intel) and any Linux with Nix. Also the only path -that supports Apple Silicon natively at the moment. +macOS (Apple Silicon and Intel) and any Linux with Nix. Use it when you +want a native binary instead of a container — the Docker path above also +runs natively on Apple Silicon. - **Prerequisites**: Make sure your Nix installation has experimental features enabled (add `experimental-features = nix-command flakes` to your `~/.config/nix/nix.conf`). @@ -60,21 +63,21 @@ that supports Apple Silicon natively at the moment. - + ```powershell Invoke-WebRequest ` - -Uri "https://raw.githubusercontent.com/TraceMachina/nativelink/v1.4.0/nativelink-config/examples/basic_cas.json5" ` + -Uri "https://raw.githubusercontent.com/TraceMachina/nativelink/v1.6.1/nativelink-config/examples/basic_cas.json5" ` -OutFile "basic_cas.json5" docker run ` -v ${PWD}/basic_cas.json5:/config ` -p 50051:50051 ` - ghcr.io/tracemachina/nativelink:v1.4.0 config + ghcr.io/tracemachina/nativelink:v1.6.1 config ``` -Native Windows support is x86_64 only. ARM64 Windows users should use -WSL2 with the Linux instructions above. +The multi-arch image covers x86_64 and ARM64 Windows machines alike. +WSL2 with the Linux instructions above works too. @@ -211,6 +214,31 @@ section below. `-v $(pwd)/basic_cas.json5:/config:Z`. +## FAQ + + + + Every action's inputs — source digests, the compiler binary, the + exact command line, declared headers — are hashed together, and the + action's outputs are stored under that hash. Anyone whose action + hashes identically gets the stored outputs back in milliseconds + instead of redoing the work. + + + The hash covers every declared input, so any difference — a header, + a compiler patch version, a leaked environment variable — produces a + new hash. That's by design: a hit is a guarantee the result is + identical. If you see misses where you expect hits, start with the + [troubleshooting](#troubleshooting) section above and the + [hermeticity FAQ](/explanations/lre#faq). + + + Yes — set both, even to the same address. Without `--remote_cache`, + Bazel calls `Execute` with an action digest it never uploaded, and + the server rejects it with `Action ... is missing from CAS`. + + + ## What's next You have a local cache running. The natural next steps: diff --git a/web/apps/docs/content/docs/index.mdx b/web/apps/docs/content/docs/index.mdx index e62b50b26..5d64ef772 100644 --- a/web/apps/docs/content/docs/index.mdx +++ b/web/apps/docs/content/docs/index.mdx @@ -15,7 +15,7 @@ section landings are accurate, page bodies are coming. If you're integrating NativeLink with an AI coding agent (Claude Code, Cursor, Copilot Workspace, Devin), start with [Getting Started → Setup](/getting-started/setup) - and then read [Why hermeticity matters](/faq/hermeticity). + and then read [why hermeticity matters](/explanations/lre#faq). ## Pick a starting point @@ -53,10 +53,6 @@ serves its job well. - [History](/explanations/history) — why NativeLink exists and what came before. -### I have a specific question - -- [FAQ](/faq/cost) — short answers to the questions we get most. - ### I want to contribute - [Contribution guidelines](/contribute/guidelines) — what we accept, @@ -70,3 +66,33 @@ serves its job well. configurable knob, autogenerated from the source. - [Glossary](/reference/glossary) - [Changelog](/reference/changelog) + +## FAQ + +Every page on this site ends with an FAQ covering the questions we +get most about that topic. The ones we get most overall: + + + + Yes — NativeLink is source-available and free for permitted + self-hosted use. Most of the monorepo is licensed under + `FSL-1.1-Apache-2.0`; a few modules (metrics, remote persistent + workers) are under the Business Source License, and a paid + [Enterprise](https://nativelink.com/pricing) tier exists if + you'd rather not run it yourself. The licensing details are in + [NativeLink on-prem](/getting-started/on-prem#faq). + + + Any client that speaks the Remote Execution API: Bazel, Buck2, Siso, + Pants, BuildStream, Goma, and plain CMake via + [recc](/getting-started/other-build-systems/cmake-recc) — with no + build-file rewrites. See + [Other build systems](/getting-started/other-build-systems). + + + Linux and Windows on x86_64 and ARM64 via the multi-arch Docker image + (`ghcr.io/tracemachina/nativelink`, v1.6.0+), and macOS (Apple + Silicon and Intel) via the Docker image or a from-source Nix build. + See [Setup](/getting-started/setup). + + diff --git a/web/apps/docs/content/docs/meta.json b/web/apps/docs/content/docs/meta.json index 3b845589d..846405094 100644 --- a/web/apps/docs/content/docs/meta.json +++ b/web/apps/docs/content/docs/meta.json @@ -8,7 +8,6 @@ "deployment", "---Learn---", "explanations", - "faq", "---Develop---", "contribute", "reference" diff --git a/web/apps/docs/content/docs/rbe/examples.mdx b/web/apps/docs/content/docs/rbe/examples.mdx index 31666a1ea..f4403d273 100644 --- a/web/apps/docs/content/docs/rbe/examples.mdx +++ b/web/apps/docs/content/docs/rbe/examples.mdx @@ -113,6 +113,30 @@ Start with cache-only. Move to full RE when the team feels the local build CPU bottleneck. Move to hybrid only when you have telemetry showing where time goes. +## FAQ + + + + The action: input digests, the command line, and the platform + requirements. The worker fetches inputs from the CAS by hash, runs + the command, and uploads outputs the same way — nothing else crosses + the wire. + + + Throughput (a fleet has more cores than any laptop), hermeticity (one + job, one environment, no stale state), and cost — build hardware goes + from the 5-10% utilization of idle CI runners to 60-80% on a shared + fleet. + + + Three levels of rigor: a Nix flake (fully hermetic — the + [LRE](/explanations/lre) path), Bazel's `rules_cc` / `rules_rust` + pinned in `MODULE.bazel` (mostly hermetic), or whatever's on `$PATH` + (quick, no guarantees). Either way, the scheduler routes actions to + matching workers via `platform_properties`. + + + ## What's next - [Nix templates](/rbe/nix-templates) — runnable testbeds for diff --git a/web/apps/docs/content/docs/rbe/local-remote-execution.mdx b/web/apps/docs/content/docs/rbe/local-remote-execution.mdx index ccc194eec..49bd9e431 100644 --- a/web/apps/docs/content/docs/rbe/local-remote-execution.mdx +++ b/web/apps/docs/content/docs/rbe/local-remote-execution.mdx @@ -107,9 +107,8 @@ build --remote_timeout=600 build --remote_executor=grpcs://TODO ``` -Replace all three `TODO`s with either your -[dev.nativelink.com](https://dev.nativelink.com) credentials or your -own cluster's endpoint. Concretely, filled in for a plain local +Replace all three `TODO`s with your own cluster's endpoint. +Concretely, filled in for a plain local cluster (no TLS, so `grpc://` instead of `grpcs://`, and no BES endpoint to point at) — this is the exact `user.bazelrc` used to produce the verified output in step 4 above: @@ -120,7 +119,7 @@ build --remote_timeout=600 build --remote_executor=grpc://127.0.0.1:50051 ``` -A [dev.nativelink.com](https://dev.nativelink.com) or self-hosted +A TLS-terminated self-hosted cluster looks the same shape, just with `grpcs://` and your cluster's real hostnames in place of `127.0.0.1:50051`, plus a `bes_backend` line if you want build results streamed there. @@ -179,6 +178,24 @@ the same CAS your local Bazel reads from. Re-run and it's a cache hit. — it also covers verifying the setup against a Kubernetes cluster. +## FAQ + + + + [Local cache and executor](/rbe/local-testing) validates the protocol + with whatever toolchain is on your `PATH` — it has no toolchain + opinion. This page pins the toolchain with Nix so local and remote + actions hash identically. Run that one first if you just want to see + `Execute` work. + + + No — the guarantee *is* the Nix pinning: every tool lives at a + content-addressed `/nix/store` path, which is what makes action + hashes match across machines. The server side, by contrast, runs fine + without Nix. + + + ## What's next - [Explanations → LRE](/explanations/lre) — the conceptual write-up. diff --git a/web/apps/docs/content/docs/rbe/local-testing.mdx b/web/apps/docs/content/docs/rbe/local-testing.mdx index 28e4dccf9..69d96d656 100644 --- a/web/apps/docs/content/docs/rbe/local-testing.mdx +++ b/web/apps/docs/content/docs/rbe/local-testing.mdx @@ -192,6 +192,22 @@ Both runs were captured against the exact config above, on macOS platform makes the worker exit immediately. +## FAQ + + + + Yes — the config exposes standard RE-API services, so Buck2, Pants, + Siso, or recc can point at `127.0.0.1:50051` the same way. The Bazel + flags are just the worked example. + + + The process counts in Bazel's output (`3 remote`, + `2 remote cache hit`) are the quick check; + `--execution_log_json_file` records the runner per action when you + need the per-action truth. + + + ## What's next - [Local Remote Execution](/rbe/local-remote-execution) — add diff --git a/web/apps/docs/content/docs/rbe/nix-templates.mdx b/web/apps/docs/content/docs/rbe/nix-templates.mdx index 0e2dcf8e5..4d23818c4 100644 --- a/web/apps/docs/content/docs/rbe/nix-templates.mdx +++ b/web/apps/docs/content/docs/rbe/nix-templates.mdx @@ -53,13 +53,11 @@ build --remote_timeout=600 build --remote_executor=grpcs://TODO ``` -Three ways to fill in `TODO` — [Local Remote Execution → Point +Two ways to fill in `TODO` — [Local Remote Execution → Point `user.bazelrc` at a real cache and executor](/rbe/local-remote-execution) has a worked, filled-in example of the file below, whichever option you pick: -- **[dev.nativelink.com](https://dev.nativelink.com)** — paste your - cloud credentials, use `grpcs://`. - **A self-hosted cluster** — your own endpoint, `grpcs://` if it terminates TLS. - **Nothing but your own machine** — drop the `s`, point both @@ -107,6 +105,23 @@ modifications: `flake.nix`, kept in sync with the `local-remote-execution` module override in `MODULE.bazel`). +## FAQ + + + + A package manager that builds everything in isolation and addresses + results by a hash of all inputs — your `clang` lives at + `/nix/store/-clang-.../bin/clang`. That hash-pinned path is + exactly what LRE needs to make action hashes identical across + machines. + + + A learning curve (an unfamiliar language, terse error messages) and + the first `nix develop`, which downloads the full toolchain — about + 5 GB for this template. After that, entering the shell is instant. + + + ## What's next - [Local Remote Execution](/rbe/local-remote-execution) — this same diff --git a/web/apps/docs/content/docs/reference/glossary.mdx b/web/apps/docs/content/docs/reference/glossary.mdx index aba4d81e7..2278f07c7 100644 --- a/web/apps/docs/content/docs/reference/glossary.mdx +++ b/web/apps/docs/content/docs/reference/glossary.mdx @@ -34,7 +34,7 @@ A content hash plus a size. Used everywhere instead of file paths. A build whose outputs depend only on its declared inputs. Same inputs → same outputs, on any machine, any time. See -[How do I make my Bazel setup hermetic?](/faq/hermeticity). +[the hermeticity FAQ](/explanations/lre#faq). ## Instance name @@ -67,15 +67,19 @@ in-flight actions. ## Worker The process that runs an action. Fetches inputs from CAS, runs the -command in a sandbox, uploads outputs to CAS. +command, uploads outputs to CAS. -## Sandbox +## Isolation -The isolated environment a worker runs an action in. On Linux, -typically `bwrap` or `landlock`; on macOS, `sandbox-exec`. The -sandbox makes hermeticity enforceable. +Keeping an action from seeing state outside its declared inputs. +On Linux, workers can isolate actions with kernel namespaces via +the `use_namespaces` and `use_mount_namespace` worker options, +which also reap zombie processes and improve hermeticity. +NativeLink doesn't currently integrate external sandboxing tools +like `bwrap`, `landlock`, or `sandbox-exec`. ## Toolchain The bundle of binaries an action needs to run — compiler, linker, -standard library, etc. See [What are toolchains?](/faq/toolchains). +standard library, etc. See +[how toolchains are provided](/rbe/examples#faq). diff --git a/web/apps/docs/content/docs/reference/nativelink-config/index.mdx b/web/apps/docs/content/docs/reference/nativelink-config/index.mdx index 62968266e..83eeadba3 100644 --- a/web/apps/docs/content/docs/reference/nativelink-config/index.mdx +++ b/web/apps/docs/content/docs/reference/nativelink-config/index.mdx @@ -5,14 +5,14 @@ full: true --- {/* AUTOGENERATED — do not edit by hand. - Source: nativelink-config @ v1.5.2 (6e63ef9a) + Source: nativelink-config @ v1.6.3 (a83dbc1a) Regenerate from web/: bun --filter @nativelink/docs gen:config-reference */} -This is the canonical NativeLink configuration reference for **v1.5.2**. +This is the canonical NativeLink configuration reference for **v1.6.3**. It is autogenerated from the Rust config crate -([`nativelink-config/src`](https://github.com/TraceMachina/nativelink/tree/v1.5.2/nativelink-config/src)) via the `build-schema` binary, so +([`nativelink-config/src`](https://github.com/TraceMachina/nativelink/tree/v1.6.3/nativelink-config/src)) via the `build-schema` binary, so it can never drift from what the binary actually deserializes. ## Top-level fields @@ -151,8 +151,8 @@ It supports the following backends: } ``` -4. **`NetApp` ONTAP S3** - `NetApp` ONTAP S3 store will use ONTAP's S3-compatible storage as a backend +4. **NetApp ONTAP S3:** + NetApp ONTAP S3 store will use ONTAP's S3-compatible storage as a backend to store files. This store is specifically configured for ONTAP's S3 requirements including custom TLS configuration, credentials management, and proper vserver configuration. @@ -180,6 +180,55 @@ It supports the following backends: } ``` +5. **Cloudflare R2:** + R2 store uses Cloudflare's R2 service as a backend. R2 speaks the + S3 API, so this is a thin wrapper that derives the account-scoped + endpoint (`https://{account_id}.r2.cloudflarestorage.com`) for you. + + **Example JSON5 config:** + ```json5 + "experimental_cloud_object_store": { + "provider": "r2", + "account_id": "a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4", + "bucket": "nativelink-cas", + "key_prefix": "test-prefix/", + "retry": { + "max_retries": 6, + "delay": 0.3, + "jitter": 0.5 + }, + "multipart_max_concurrent_uploads": 10 + } + ``` + +6. **Oracle Cloud Infrastructure (OCI) Object Storage:** + OCI store uses Oracle Cloud Infrastructure's S3-compatible Object + Storage API. The path-style endpoint is derived from your Object + Storage `namespace` and `region` as + `https://{namespace}.compat.objectstorage.{region}.oci.customer-oci.com`. + Authenticate with a Customer Secret Key (Access Key/Secret Key pair + created under User Settings -> Customer secret keys in the OCI + console); the secret cannot be retrieved after generation, so read + it from an env var via shellexpand. + + **Example JSON5 config:** + ```json5 + "experimental_cloud_object_store": { + "provider": "oci", + "namespace": "your-object-storage-namespace", + "region": "us-phoenix-1", + "bucket": "nativelink-cas", + "access_key_id": "oci_access_key_id", + "secret_access_key": "oci_secret_access_key", + "key_prefix": "test-prefix/", + "retry": { + "max_retries": 6, + "delay": 0.3, + "jitter": 0.5 + } + } + ``` + **Type:** [ExperimentalCloudObjectSpec](#experimentalcloudobjectspec) ### `ontap_s3_existence_cache` @@ -402,7 +451,7 @@ On uploads it will mirror data to both `fast` and `slow` stores. WARNING: If you need data to always exist in the `slow` store for something like remote execution, be careful because this store will never check to see if the objects exist in the -`slow` store if it exists in the `fast` store (i.e., it assumes +`slow` store if it exists in the `fast` store (i.e. it assumes that if an object exists in the `fast` store it will exist in the `slow` store). @@ -503,7 +552,7 @@ in one store and large objects in another store. This should only be used if the size field is the real size of the content, in other words, don't use on AC (Action Cache) stores. Any store where you can safely use `VerifySpec.verify_size = true`, this store should be safe -to use (i.e., CAS stores). +to use (i.e. CAS stores). **Example JSON5 config:** ```json5 @@ -527,7 +576,7 @@ to use (i.e., CAS stores). ### `grpc` -This store will pass-through calls to another GRPC store. This store +This store will pass-through calls to another gRPC store. This store is not designed to be used as a sub-store of another store, but it does satisfy the interface and will likely work. @@ -569,6 +618,12 @@ Pairs well with `SizePartitioning` and/or `FastSlow` stores. Ideal for accepting small object sizes as most Redis store services have a max file upload of between 256Mb-512Mb. +If you are using Redis together with any stores above it +e.g. existence cache, you will need to configure `notify-keyspace-events` +to `KA` as per [https://redis.io/docs/latest/develop/pubsub/keyspace-notifications/#configuration](https://redis.io/docs/latest/develop/pubsub/keyspace-notifications/#configuration) +in order for us to get eviction events. Failing to do so will get you +log messages complaining about it, as well as errors like [https://github.com/TraceMachina/nativelink/issues/2436](https://github.com/TraceMachina/nativelink/issues/2436) + **Example JSON5 config:** ```json5 "redis_store": { @@ -688,7 +743,7 @@ Plus exactly one of the following variants (the key selects the variant): | Field | Type | Required | Default | Description | | --- | --- | --- | --- | --- | | `cache_type` | string | Yes | — | Low-cardinality cache type label for metrics, for example `cas` or `ac`. | -| `backend` | [StoreSpec](#storespec) | Yes | — | Store to wrap with cache operation metrics. | +| `backend` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | Store to wrap with cache operation metrics. | ## MemorySpec @@ -712,7 +767,7 @@ See [`experimental_cloud_object_store`](#experimental_cloud_object_store-1) for | Field | Type | Required | Default | Description | | --- | --- | --- | --- | --- | -| `backend` | [StoreSpec](#storespec) | Yes | — | The underlying store wrap around. All content will first flow through self before forwarding to backend. In the event there is an error detected in self, the connection to the backend will be terminated, and early termination should always cause updates to fail on the backend. | +| `backend` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | The underlying store wrap around. All content will first flow through self before forwarding to backend. In the event there is an error detected in self, the connection to the backend will be terminated, and early termination should always cause updates to fail on the backend. | | `verify_size` | boolean | — | `false` | If set the store will verify the size of the data before accepting an upload of data. | | `verify_hash` | boolean | — | `false` | If the data should be hashed and verify that the key matches the computed hash. The hash function is automatically determined based request and if not set will use the global default. | @@ -720,22 +775,22 @@ See [`experimental_cloud_object_store`](#experimental_cloud_object_store-1) for | Field | Type | Required | Default | Description | | --- | --- | --- | --- | --- | -| `backend` | [StoreSpec](#storespec) | Yes | — | The underlying store that will have it's results validated before sending to client. | -| `cas_store` | [StoreSpec](#storespec) | Yes | — | When a request is made, the results are decoded and all output digests/files are verified to exist in this CAS store before returning success. | +| `backend` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | The underlying store that will have it's results validated before sending to client. | +| `cas_store` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | When a request is made, the results are decoded and all output digests/files are verified to exist in this CAS store before returning success. | ## CompressionSpec | Field | Type | Required | Default | Description | | --- | --- | --- | --- | --- | -| `backend` | [StoreSpec](#storespec) | Yes | — | The underlying store wrap around. All content will first flow through self before forwarding to backend. In the event there is an error detected in self, the connection to the backend will be terminated, and early termination should always cause updates to fail on the backend. | +| `backend` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | The underlying store wrap around. All content will first flow through self before forwarding to backend. In the event there is an error detected in self, the connection to the backend will be terminated, and early termination should always cause updates to fail on the backend. | | `compression_algorithm` | [CompressionAlgorithm](#compressionalgorithm) | Yes | — | The compression algorithm to use. | ## DedupSpec | Field | Type | Required | Default | Description | | --- | --- | --- | --- | --- | -| `index_store` | [StoreSpec](#storespec) | Yes | — | Store used to store the index of each dedup slice. This store should generally be fast and small. | -| `content_store` | [StoreSpec](#storespec) | Yes | — | The store where the individual chunks will be uploaded. This store should generally be the slower & larger store. | +| `index_store` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | Store used to store the index of each dedup slice. This store should generally be fast and small. | +| `content_store` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | The store where the individual chunks will be uploaded. This store should generally be the slower & larger store. | | `min_size` | integer (uint32) | — | 64k | Minimum size that a chunk will be when slicing up the content. Note: This setting can be increased to improve performance because it will actually not check this number of bytes when deciding where to partition the data. | | `normal_size` | integer (uint32) | — | 256k | A best-effort attempt will be made to keep the average size of the chunks to this number. It is not a guarantee, but a slight attempt will be made. | | `max_size` | integer (uint32) | — | 512k | Maximum size a chunk is allowed to be. | @@ -745,16 +800,16 @@ See [`experimental_cloud_object_store`](#experimental_cloud_object_store-1) for | Field | Type | Required | Default | Description | | --- | --- | --- | --- | --- | -| `backend` | [StoreSpec](#storespec) | Yes | — | The underlying store wrap around. All content will first flow through self before forwarding to backend. In the event there is an error detected in self, the connection to the backend will be terminated, and early termination should always cause updates to fail on the backend. | +| `backend` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | The underlying store wrap around. All content will first flow through self before forwarding to backend. In the event there is an error detected in self, the connection to the backend will be terminated, and early termination should always cause updates to fail on the backend. | | `eviction_policy` | [EvictionPolicy](#evictionpolicy) | — | — | Policy used to evict items out of the store. Failure to set this value will cause items to never be removed from the store causing infinite memory usage. | ## FastSlowSpec | Field | Type | Required | Default | Description | | --- | --- | --- | --- | --- | -| `fast` | [StoreSpec](#storespec) | Yes | — | Fast store that will be attempted to be contacted before reaching out to the `slow` store. | +| `fast` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | Fast store that will be attempted to be contacted before reaching out to the `slow` store. | | `fast_direction` | [StoreDirection](#storedirection) | — | `"both"` | How to handle the fast store. This can be useful to set to Get for worker nodes such that results are persisted to the slow store only. | -| `slow` | [StoreSpec](#storespec) | Yes | — | If the object does not exist in the `fast` store it will try to get it from this store. | +| `slow` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | If the object does not exist in the `fast` store it will try to get it from this store. | | `slow_direction` | [StoreDirection](#storedirection) | — | `"both"` | How to handle the slow store. This can be useful if creating a diode and you wish to have an upstream read only store. | | `bypass_dedup_threshold_bytes` | integer (uint64) | — | disabled (0) | Reads of blobs at or above this size skip the leader/follower dedup map and stream straight from the slow store without populating the fast tier. `0` (the default) disables the bypass: every read goes through dedup, matching the prior behaviour. Enable it by setting a threshold — 256 MiB is a reasonable starting point for backends where large-blob dedup is a net loss (followers tend to time out anyway), but the right value is workload-dependent. | @@ -768,12 +823,13 @@ See [`experimental_cloud_object_store`](#experimental_cloud_object_store-1) for | Field | Type | Required | Default | Description | | --- | --- | --- | --- | --- | -| `content_path` | string | Yes | — | Path on the system where to store the actual content. This is where the bulk of the data will be placed. On service startup this folder will be scanned and all files will be added to the cache. In the event one of the files doesn't match the criteria, the file will be deleted. | -| `temp_path` | string | Yes | — | A temporary location of where files that are being uploaded or deleted will be placed while the content cannot be guaranteed to be accurate. This location must be on the same block device as `content_path` so atomic moves can happen (i.e., move without copy). All files in this folder will be deleted on every startup. | +| `content_path` | string | Yes | — | Path on the system where to store the actual content. This is where the bulk of the data will be placed. On service boot this folder will be scanned and all files will be added to the cache. In the event one of the files doesn't match the criteria, the file will be deleted. | +| `temp_path` | string | Yes | — | A temporary location of where files that are being uploaded or deleted will be placed while the content cannot be guaranteed to be accurate. This location must be on the same block device as `content_path` so atomic moves can happen (i.e. move without copy). All files in this folder will be deleted on every startup. | | `read_buffer_size` | integer (uint32) | — | 32k | Buffer size to use when reading files. Generally this should be left to the default value except for testing. | | `eviction_policy` | [EvictionPolicy](#evictionpolicy) | — | — | Policy used to evict items out of the store. Failure to set this value will cause items to never be removed from the store causing infinite memory usage. | | `block_size` | integer (uint64) | — | 4kb | The block size of the filesystem for the running machine value is used to determine an entry's actual size on disk consumed For a 4KB block size filesystem, a 1B file actually consumes 4KB | | `max_concurrent_writes` | integer (uint) | — | unlimited | Maximum number of concurrent write operations allowed. Each write involves streaming data to a temp file and calling `sync_all()`, which can saturate disk I/O when many writes happen simultaneously. Limiting concurrency prevents disk saturation from blocking the async runtime. A value of 0 means unlimited (no concurrency limit). | +| `evict_page_cache` | boolean | — | false | When true, advise the kernel to drop the page cache for each blob after it is written or read (`posix_fadvise` with `POSIX_FADV_DONTNEED`). On real filesystems this takes a globally serialized, all-CPU kernel path that stalls on many-core hosts, and it evicts the page-cache tier that fast-disk deployments rely on. Leave off unless you specifically want to keep this store's I/O out of the page cache. | ## RefSpec @@ -786,23 +842,24 @@ See [`experimental_cloud_object_store`](#experimental_cloud_object_store-1) for | Field | Type | Required | Default | Description | | --- | --- | --- | --- | --- | | `size` | integer (uint64) | Yes | — | Size to partition the data on. | -| `lower_store` | [StoreSpec](#storespec) | Yes | — | Store to send data when object is < (less than) size. | -| `upper_store` | [StoreSpec](#storespec) | Yes | — | Store to send data when object is >= (less than eq) size. | +| `lower_store` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | Store to send data when object is < (less than) size. | +| `upper_store` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | Store to send data when object is >= (less than eq) size. | ## GrpcSpec | Field | Type | Required | Default | Description | | --- | --- | --- | --- | --- | -| `instance_name` | string | — | `""` | Instance name for GRPC calls. Proxy calls will have the `instance_name` changed to this. | +| `instance_name` | string | — | `""` | Instance name for gRPC calls. Proxy calls will have the `instance_name` changed to this. | | `endpoints` | array of [GrpcEndpoint](#grpcendpoint) | Yes | — | The endpoint of the grpc connection. | | `store_type` | [StoreType](#storetype) | Yes | — | The type of the upstream store, this ensures that the correct server calls are made. | | `retry` | [Retry](#retry) | — | — | Retry configuration to use when a network request fails. | | `max_concurrent_requests` | integer (uint) | — | `0` | Limit the number of simultaneous upstream requests to this many. A value of zero is treated as unlimited. If the limit is reached the request is queued. | -| `connections_per_endpoint` | integer (uint) | — | `0` | The number of connections to make to each specified endpoint to balance the load over multiple TCP connections. Default 1. | +| `connections_per_endpoint` | integer (uint) | — | 1 | The number of connections to make to each specified endpoint to balance the load over multiple TCP connections. | | `rpc_timeout_s` | integer (uint64) | — | 0 (disabled) | Maximum time (seconds) allowed for a single RPC request (e.g. a `ByteStream.Write` call) before it is cancelled. | | `use_legacy_resource_names` | boolean | — | false | Use legacy `ByteStream` resource name format, omitting the digest function component from the path. | | `headers` | map of string to string | — | — | Static headers to attach to every outgoing gRPC request sent to this store's upstream endpoints. Useful for fixed authentication tokens (e.g. `{"authorization": "Bearer "}`) and other static metadata. | | `forward_headers` | array of string | — | — | Header names to forward from the incoming client request to every outgoing upstream request. The header value is taken from the client request that triggered this store operation. Use this to pass through dynamic credentials such as JWT tokens sent by build clients. | +| `experimental_read_batching` | [GrpcReadBatchingConfig](#grpcreadbatchingconfig) | — | unset (disabled) | Optional and experimental: coalesce small-blob reads into `BatchReadBlobs` RPCs instead of issuing one `ByteStream` `Read` stream per blob. Each `ByteStream` read carries a fixed per-RPC cost, so batching many small reads into a single `BatchReadBlobs` request can dramatically reduce read latency for small blobs. | ## RedisSpec @@ -820,7 +877,7 @@ See [`experimental_cloud_object_store`](#experimental_cloud_object_store-1) for | `health_check_timeout_ms` | integer (uint64) | — | 4000 (4 seconds) | Per-call ceiling for the `check_health` PING in milliseconds. | | `read_chunk_size` | integer (uint) | — | 64KiB | The amount of data to read from the Redis server at a time. This is used to limit the amount of memory used when reading large objects from the Redis server as well as limiting the amount of time a single read operation can take. | | `connection_pool_size` | integer (uint) | — | 3 | The number of connections to keep open to the Redis servers. | -| `max_chunk_uploads_per_update` | integer (uint) | — | 10 | The maximum number of upload chunks to allow per update. This is used to limit the amount of memory used when uploading large objects to the Redis server. A good rule of thumb is to think of the data as: `AVAIL_MEMORY / (read_chunk_size * max_chunk_uploads_per_update) = THORETICAL_MAX_CONCURRENT_UPLOADS` (note: it is a good idea to divide `AVAIL_MAX_MEMORY` by ~10 to account for other memory usage) | +| `max_chunk_uploads_per_update` | integer (uint) | — | 10 | The maximum number of upload chunks to allow per update. This is used to limit the amount of memory used when uploading large objects to the Redis server. A good rule of thumb is to think of the data as: `AVAIL_MEMORY / (read_chunk_size * max_chunk_uploads_per_update) = THORETICAL_MAX_CONCURRENT_UPLOADS` (note: it's a good idea to divide `AVAIL_MAX_MEMORY` by ~10 to account for other memory usage) | | `scan_count` | integer (uint) | — | 10000 | The COUNT value passed when scanning keys in Redis. This is used to hint the amount of work that should be done per response. | | `retry` | [Retry](#retry) | — | — | Retry configuration to use when a network request fails. | | `max_client_permits` | integer (uint) | — | 500 | Maximum number of permitted actions to the Redis store at any one time This stops problems with timeouts due to many, many inflight actions | @@ -855,10 +912,12 @@ Configuration for `ExperimentalMongoDB` store. | Field | Type | Required | Default | Description | | --- | --- | --- | --- | --- | -| `name` | string | — | {Index position in the workers list} | Name of the worker. This is give a more friendly name to a worker for logging and metric publishing. This is also the prefix of the worker id (i.e., "{name}{uuidv6}"). | +| `name` | string | — | {Index position in the workers list} | Name of the worker. This is give a more friendly name to a worker for logging and metric publishing. This is also the prefix of the worker id (i.e. "{name}{uuidv6}"). | | `worker_api_endpoint` | [EndpointConfig](#endpointconfig) | Yes | — | Endpoint which the worker will connect to the scheduler's `WorkerApiService`. | -| `max_action_timeout` | integer (uint) | — | 20 minutes | The maximum time an action is allowed to run. If a task requests for a timeout longer than this time limit, the task will be rejected. Value in seconds. | -| `max_upload_timeout` | integer (uint) | — | 10 minutes | Maximum time allowed for uploading action results to CAS after execution completes. If upload takes longer than this, the action fails with `DeadlineExceeded` and may be retried by the scheduler. Value in seconds. | +| `max_action_timeout_s` | integer (uint) | — | 20 minutes | The maximum time an action is allowed to run. If a task requests for a timeout longer than this time limit, the task will be rejected. Value in seconds. | +| `max_upload_timeout_s` | integer (uint) | — | 10 minutes | Maximum time allowed for uploading action results to CAS after execution completes. If upload takes longer than this, the action fails with `DeadlineExceeded` and may be retried by the scheduler. Value in seconds. | +| `max_cleanup_wait_s` | integer (uint) | — | 30 seconds | Maximum time to wait for action directory cleanup before timing out. Value in seconds. | +| `max_cleanup_backoff_ms` | integer (uint) | — | 500 milliseconds | Maximum backoff duration for exponential backoff when waiting for cleanup. Value in milliseconds. | | `max_inflight_tasks` | integer (uint64) | — | 0 (infinite tasks) | Maximum number of inflight tasks this worker can cope with. | | `timeout_handled_externally` | boolean | — | false (`NativeLink` fully handles timeouts) | If timeout is handled in `entrypoint` or another wrapper script. If set to true `NativeLink` will not honor the timeout the action requested and instead will always force kill the action after `max_action_timeout` has been reached. If this is set to false, the smaller value of the action's timeout and `max_action_timeout` will be used to which `NativeLink` will kill the action. | | `entrypoint` | string | — | {Use the command from the job request} | The command to execute on every execution request. This will be parsed as a command + arguments (not shell). Example: "run.sh" and a job with command: "sleep 5" will result in a command like: "run.sh sleep 5". | @@ -885,6 +944,7 @@ Configuration for `ExperimentalMongoDB` store. | `allocation_strategy` | [WorkerAllocationStrategy](#workerallocationstrategy) | — | `"least_recently_used"` | The strategy used to assign workers jobs. | | `experimental_backend` | [ExperimentalSimpleSchedulerBackend](#experimentalsimpleschedulerbackend) | — | memory | The storage backend to use for the scheduler. | | `worker_match_logging_interval_s` | integer (int64) | — | `10` | Every N seconds, do logging of worker matching e.g. "worker busy", "can't find any worker" Defaults to 10s. Can be set to `-1` to disable | +| `fallback_match_interval_s` | integer (int64) | — | `5` | Every N seconds, run a worker matching pass even if no task or worker change notification arrived. This is a safety net for missed notifications and for scheduler backends with eventually consistent searches (for example Redis), where an operation that was re-queued may not be visible to the search triggered by its own notification. Without this, such an operation can stay queued until an unrelated event triggers another matching pass. Defaults to 5s. Zero or any negative value disables it. | ## SchedulerGrpcSpec @@ -940,7 +1000,7 @@ Listener for HTTP/HTTPS/HTTP2 sockets. | --- | --- | --- | --- | --- | | `cas` | array of [CasServiceConfig](#casserviceconfig) | — | — | The Content Addressable Storage (CAS) backend config. The key is the `instance_name` used in the protocol and the value is the underlying CAS store config. | | `ac` | array of [ActionCacheServiceConfig](#actioncacheserviceconfig) | — | — | The Action Cache (AC) backend config. The key is the `instance_name` used in the protocol and the value is the underlying AC store config. | -| `capabilities` | array of [CapabilitiesServiceConfig](#capabilitiesserviceconfig) | — | — | Capabilities service is required in order to use most of the bazel protocol. This service is used to provide the supported features and versions of this bazel GRPC service. | +| `capabilities` | array of [CapabilitiesServiceConfig](#capabilitiesserviceconfig) | — | — | Capabilities service is required in order to use most of the bazel protocol. This service is used to provide the supported features and versions of this bazel gRPC service. | | `execution` | array of [ExecutionServiceConfig](#executionserviceconfig) | — | — | The remote execution service configuration. NOTE: This service is under development and is currently just a place holder. | | `bytestream` | array of [ByteStreamServiceConfig](#bytestreamserviceconfig) | — | — | This is the service used to stream data to and from the CAS. Bazel's protocol strongly encourages users to use this streaming interface to interact with the CAS when the data is large. | | `fetch` | array of [FetchServiceConfig](#fetchserviceconfig) | — | — | These two are collectively the Remote Asset protocol, but it's defined as two separate services | @@ -970,702 +1030,133 @@ Listener for HTTP/HTTPS/HTTP2 sockets. | `"sha256"` | Use the sha256 hash function. [https://en.wikipedia.org/wiki/SHA-2](https://en.wikipedia.org/wiki/SHA-2) | | `"blake3"` | Use the blake3 hash function. [https://en.wikipedia.org/wiki/BLAKE_(hash_function)](https://en.wikipedia.org/wiki/BLAKE_(hash_function)) | -## StoreSpec - -Plus exactly one of the following variants (the key selects the variant): +## EvictionPolicy -### `cache_metrics` +Eviction policy always works on LRU (Least Recently Used). Any time an entry +is touched it updates the timestamp. Inserts and updates will execute the +eviction policy removing any expired entries and/or the oldest entries +until the store size becomes smaller than `max_bytes`. -Cache metrics store wraps another store and emits low-cardinality -OpenTelemetry cache operation metrics for the wrapped store. +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `max_bytes` | integer (uint) | — | 0 | Maximum number of bytes before eviction takes place. Zero means never evict based on size. | +| `evict_bytes` | integer (uint) | — | 0 | When eviction starts based on hitting `max_bytes`, continue until `max_bytes - evict_bytes` is met to create a low watermark. This stops operations from thrashing when the store is close to the limit. | +| `max_seconds` | integer (uint32) | — | 0 | Maximum number of seconds for an entry to live since it was last accessed before it is evicted. Zero means never evict based on time. | +| `max_count` | integer (uint64) | — | 0 | Maximum size of the store before an eviction takes place. Zero means never evict based on count. | -This wrapper is opt-in. Stores that are not explicitly wrapped by -`cache_metrics` are constructed exactly as they are without this -wrapper and do not pay its hot-path timing or recording cost. +## Retry -**Example JSON5 config:** -```json5 -"cache_metrics": { - "cache_type": "cas", - "backend": { - "filesystem": { - "content_path": "~/.cache/nativelink/content_path-cas", - "temp_path": "~/.cache/nativelink/tmp_path-cas" - } - } +Retry configuration. This configuration is exponential and each iteration +a jitter as a percentage is applied of the calculated delay. For example: +```haskell +Retry{ + max_retries: 7, + delay: 0.1, + jitter: 0.5, } ``` +will result in: -**Type:** [CacheMetricsSpec](#cachemetricsspec) +| Attempt | Delay | +| --- | --- | +| 1 | 0 ms | +| 2 | 75 to 125 ms | +| 3 | 150 to 250 ms | +| 4 | 300 to 500 ms | +| 5 | 600 ms to 1 s | +| 6 | 1.2 to 2 s | +| 7 | 2.4 to 4 s | +| 8 | 4.8 to 8 s | -### `memory` +The total delay is additive, so this example produces 9.525 to 15.875 s of total delay for a single request. -Memory store will store all data in a hash map in memory. +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `max_retries` | integer (uint) | — | `0` | Maximum number of retries until retrying stops. Setting this to zero will always attempt 1 time, but not retry. | +| `delay` | number (float) | — | `0` | Delay in seconds for exponential back off. | +| `jitter` | number (float) | — | `0` | Amount of jitter to add as a percentage in decimal form. This will change the formula like: | +| `retry_on_errors` | array of [ErrorCode](#errorcode) | — | — | A list of error codes to retry on, if this isn't set then the default error codes to retry on are used. These default codes are the most likely to be non-permanent: `Unknown`; `Cancelled`; `DeadlineExceeded`; `ResourceExhausted`; `Aborted`; `Internal`; `Unavailable`; `DataLoss` | -**Example JSON5 config:** -```json5 -"memory": { - "eviction_policy": { - "max_bytes": "10mb", - } -} -``` +## ExperimentalOntapS3Spec -**Type:** [MemorySpec](#memoryspec) +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `endpoint` | string | Yes | — | | +| `vserver_name` | string | Yes | — | | +| `bucket` | string | Yes | — | | +| `root_certificates` | string | — | — | | +| `key_prefix` | string | — | — | If you wish to prefix the location in the bucket. If None, no prefix will be used. | +| `retry` | [Retry](#retry) | — | — | Retry configuration to use when a network request fails. | +| `consider_expired_after_s` | integer (uint32) | — | 0 | If the number of seconds since the `last_modified` time of the object is greater than this value, the object will not be considered "existing". This allows for external tools to delete objects that have not been uploaded in a long time. If a client receives a `NotFound` the client should re-upload the object. | +| `max_retry_buffer_per_request` | integer (uint) | — | 5MB | The maximum buffer size to retain in case of a retryable error during upload. Setting this to zero will disable upload buffering; this means that in the event of a failure during upload, the entire upload will be aborted and the client will likely receive an error. | +| `multipart_max_concurrent_uploads` | integer (uint) | — | 10 | Maximum number of concurrent `UploadPart` requests per `MultipartUpload`. | +| `insecure_allow_http` | boolean | — | false | Allow unencrypted HTTP connections. Only use this for local testing. | +| `disable_http2` | boolean | — | false | Disable HTTP/2 connections and only use HTTP/1.1. Default client configuration will have HTTP/1.1 and HTTP/2 enabled for connection schemes. HTTP/2 should be disabled if environments have poor support or performance related to HTTP/2. Safe to keep default unless underlying network environment, S3, or GCS API servers specify otherwise. | -### `experimental_cloud_object_store` +## CompressionAlgorithm -A generic blob store that will store files on the cloud -provider. This configuration will never delete files, so you are -responsible for purging old files in other ways. -It supports the following backends: +Plus exactly one of the following variants (the key selects the variant): -1. **Amazon S3:** - S3 store will use Amazon's S3 service as a backend to store - the files. This configuration can be used to share files - across multiple instances. Uses system certificates for TLS - verification via `rustls-platform-verifier`. +### `lz4` - **Example JSON5 config:** - ```json5 - "experimental_cloud_object_store": { - "provider": "aws", - "region": "eu-north-1", - "bucket": "crossplane-bucket-af79aeca9", - "key_prefix": "test-prefix-index/", - "retry": { - "max_retries": 6, - "delay": 0.3, - "jitter": 0.5 - }, - "multipart_max_concurrent_uploads": 10 - } - ``` +LZ4 compression algorithm is extremely fast for compression and +decompression, however does not perform very well in compression +ratio. In most cases build artifacts are highly compressible, however +lz4 is quite good at aborting early if the data is not deemed very +compressible. -2. **Google Cloud Storage:** - GCS store uses Google's GCS service as a backend to store - the files. This configuration can be used to share files - across multiple instances. +see: [https://lz4.github.io/lz4/](https://lz4.github.io/lz4/) - **Example JSON5 config:** - ```json5 - "experimental_cloud_object_store": { - "provider": "gcs", - "bucket": "test-bucket", - "key_prefix": "test-prefix-index/", - "retry": { - "max_retries": 6, - "delay": 0.3, - "jitter": 0.5 - }, - "multipart_max_concurrent_uploads": 10 - } - ``` +**Type:** [Lz4Config](#lz4config) -3. **Azure Blob Store:** - Azure Blob store will use Microsoft's Azure Blob service as a - backend to store the files. This configuration can be used to - share files across multiple instances. +## StoreDirection - **Example JSON5 config:** - ```json5 - "experimental_cloud_object_store": { - "provider": "azure", - "account_name": "cloudshell1393657559", - "container": "simple-test-container", - "key_prefix": "folder/", - "retry": { - "max_retries": 6, - "delay": 0.3, - "jitter": 0.5 - }, - "multipart_max_concurrent_uploads": 10 - } - ``` +| Value | Description | +| --- | --- | +| `"both"` | The store operates normally and all get and put operations are handled by it. | +| `"update"` | Update operations will cause persistence to this store, but Get operations will be ignored. This only makes sense on the fast store as the slow store will never get written to on Get anyway. | +| `"get"` | Get operations will cause persistence to this store, but Update operations will be ignored. | +| `"read_only"` | Operate as a read only store, only really makes sense if there's another way to write to it. | -4. **`NetApp` ONTAP S3** - `NetApp` ONTAP S3 store will use ONTAP's S3-compatible storage as a backend - to store files. This store is specifically configured for ONTAP's S3 requirements - including custom TLS configuration, credentials management, and proper vserver - configuration. +## ShardConfig - This store uses AWS environment variables for credentials: - - `AWS_ACCESS_KEY_ID` - - `AWS_SECRET_ACCESS_KEY` - - `AWS_DEFAULT_REGION` +Configuration for an individual shard of the store. - **Example JSON5 config:** - ```json5 - "experimental_cloud_object_store": { - "provider": "ontap", - "endpoint": "https://ontap-s3-endpoint:443", - "vserver_name": "your-vserver", - "bucket": "your-bucket", - "root_certificates": "/path/to/certs.pem", // Optional - "key_prefix": "test-prefix/", // Optional - "retry": { - "max_retries": 6, - "delay": 0.3, - "jitter": 0.5 - }, - "multipart_max_concurrent_uploads": 10 - } - ``` +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `store` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | Store to shard the data to. | +| `weight` | integer (uint32) | — | 1 | The weight of the store. This is used to determine how much data should be sent to the store. The actual percentage is the sum of all the store's weights divided by the individual store's weight. | -**Type:** [ExperimentalCloudObjectSpec](#experimentalcloudobjectspec) +## GrpcEndpoint -### `ontap_s3_existence_cache` +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `address` | string | Yes | — | The endpoint address (i.e. `grpc(s)://example.com:443`). | +| `tls_config` | [ClientTlsConfig](#clienttlsconfig) | — | — | The TLS configuration to use to connect to the endpoint (if grpcs). | +| `concurrency_limit` | integer (uint) | — | — | The maximum concurrency to allow on this endpoint. | +| `connect_timeout_s` | integer (uint64) | — | 30 seconds | Timeout for establishing a TCP connection to the endpoint (seconds). | +| `tcp_keepalive_s` | integer (uint64) | — | 30 seconds | TCP keepalive interval (seconds). Sends TCP keepalive probes at this interval to detect dead connections at the OS level. | +| `http2_keepalive_interval_s` | integer (uint64) | — | 30 seconds | HTTP/2 keepalive interval (seconds). Sends HTTP/2 PING frames at this interval to detect dead connections at the application level. | +| `http2_keepalive_timeout_s` | integer (uint64) | — | 20 seconds | HTTP/2 keepalive timeout (seconds). If a PING response is not received within this duration, the connection is considered dead. | -ONTAP S3 Existence Cache provides a caching layer on top of the ONTAP S3 store -to optimize repeated existence checks. It maintains an in-memory cache of object -digests and periodically syncs this cache to disk for persistence. +## StoreType -The cache helps reduce latency for repeated calls to check object existence, -while still ensuring eventual consistency with the underlying ONTAP S3 store. +| Value | Description | +| --- | --- | +| `"cas"` | The store is content addressable storage. | +| `"ac"` | The store is an action cache. | -Example JSON5 config: -```json5 -"ontap_s3_existence_cache": { - "index_path": "/path/to/cache/index.json", - "sync_interval_seconds": 300, - "backend": { - "endpoint": "https://ontap-s3-endpoint:443", - "vserver_name": "your-vserver", - "bucket": "your-bucket", - "key_prefix": "test-prefix/" - } -} -``` +## GrpcReadBatchingConfig -**Type:** [OntapS3ExistenceCacheSpec](#ontaps3existencecachespec) +Configuration for experimental small-blob read coalescing in a gRPC +store. See [`GrpcSpec::experimental_read_batching`]. -### `verify` - -Verify store is used to apply verifications to an underlying -store implementation. It is strongly encouraged to validate -as much data as you can before accepting data from a client, -failing to do so may cause the data in the store to be -populated with invalid data causing all kinds of problems. - -The suggested configuration is to have the CAS validate the -hash and size and the AC validate nothing. - -**Example JSON5 config:** -```json5 -"verify": { - "backend": { - "memory": { - "eviction_policy": { - "max_bytes": "500mb" - } - }, - }, - "verify_size": true, - "verify_hash": true -} -``` - -**Type:** [VerifySpec](#verifyspec) - -### `completeness_checking` - -Completeness checking store verifies if the -output files & folders exist in the CAS before forwarding -the request to the underlying store. -Note: This store should only be used on AC stores. - -**Example JSON5 config:** -```json5 -"completeness_checking": { - "backend": { - "filesystem": { - "content_path": "~/.cache/nativelink/content_path-ac", - "temp_path": "~/.cache/nativelink/tmp_path-ac", - "eviction_policy": { - "max_bytes": "500mb", - } - } - }, - "cas_store": { - "ref_store": { - "name": "CAS_MAIN_STORE" - } - } -} -``` - -**Type:** [CompletenessCheckingSpec](#completenesscheckingspec) - -### `compression` - -A compression store that will compress the data inbound and -outbound. There will be a non-trivial cost to compress and -decompress the data, but in many cases if the final store is -a store that requires network transport and/or storage space -is a concern it is often faster and more efficient to use this -store before those stores. - -**Example JSON5 config:** -```json5 -"compression": { - "compression_algorithm": { - "lz4": {} - }, - "backend": { - "filesystem": { - "content_path": "/tmp/nativelink/data/content_path-cas", - "temp_path": "/tmp/nativelink/data/tmp_path-cas", - "eviction_policy": { - "max_bytes": "2gb", - } - } - } -} -``` - -**Type:** [CompressionSpec](#compressionspec) - -### `dedup` - -A dedup store will take the inputs and run a rolling hash -algorithm on them to slice the input into smaller parts then -run a sha256 algorithm on the slice and if the object doesn't -already exist, upload the slice to the `content_store` using -a new digest of just the slice. Once all parts exist, an -Action-Cache-like digest will be built and uploaded to the -`index_store` which will contain a reference to each -chunk/digest of the uploaded file. Downloading a request will -first grab the index from the `index_store`, and forward the -download content of each chunk as if it were one file. - -This store is exceptionally good when the following conditions -are met: -* Content is mostly the same (inserts, updates, deletes are ok) -* Content is not compressed or encrypted -* Uploading or downloading from `content_store` is the bottleneck. - -Note: This store pairs well when used with `CompressionSpec` as -the `content_store`, but never put `DedupSpec` as the backend of -`CompressionSpec` as it will negate all the gains. - -Note: When running `.has()` on this store, it will only check -to see if the entry exists in the `index_store` and not check -if the individual chunks exist in the `content_store`. - -**Example JSON5 config:** -```json5 -"dedup": { - "index_store": { - "memory": { - "eviction_policy": { - "max_bytes": "1GB", - } - } - }, - "content_store": { - "compression": { - "compression_algorithm": { - "lz4": {} - }, - "backend": { - "fast_slow": { - "fast": { - "memory": { - "eviction_policy": { - "max_bytes": "500MB", - } - } - }, - "slow": { - "filesystem": { - "content_path": "/tmp/nativelink/data/content_path-content", - "temp_path": "/tmp/nativelink/data/tmp_path-content", - "eviction_policy": { - "max_bytes": "2gb" - } - } - } - } - } - } - } -} -``` - -**Type:** [DedupSpec](#dedupspec) - -### `existence_cache` - -Existence store will wrap around another store and cache calls -to has so that subsequent `has_with_results` calls will be -faster. This is useful for cases when you have a store that -is slow to respond to has calls. -Note: This store should only be used on CAS stores. - -**Example JSON5 config:** -```json5 -"existence_cache": { - "backend": { - "memory": { - "eviction_policy": { - "max_bytes": "500mb", - } - } - }, - // Note this is the existence store policy, not the backend policy - "eviction_policy": { - "max_seconds": 100, - } -} -``` - -**Type:** [ExistenceCacheSpec](#existencecachespec) - -### `fast_slow` - -`FastSlow` store will first try to fetch the data from the `fast` -store and then if it does not exist try the `slow` store. -When the object does exist in the `slow` store, it will copy -the data to the `fast` store while returning the data. -This store should be thought of as a store that "buffers" -the data to the `fast` store. -On uploads it will mirror data to both `fast` and `slow` stores. - -WARNING: If you need data to always exist in the `slow` store -for something like remote execution, be careful because this -store will never check to see if the objects exist in the -`slow` store if it exists in the `fast` store (i.e., it assumes -that if an object exists in the `fast` store it will exist in -the `slow` store). - -***Example JSON5 config:*** -```json5 -"fast_slow": { - "fast": { - "filesystem": { - "content_path": "/tmp/nativelink/data/content_path-index", - "temp_path": "/tmp/nativelink/data/tmp_path-index", - "eviction_policy": { - "max_bytes": "500mb", - } - } - }, - "slow": { - "filesystem": { - "content_path": "/tmp/nativelink/data/content_path-index", - "temp_path": "/tmp/nativelink/data/tmp_path-index", - "eviction_policy": { - "max_bytes": "500mb", - } - } - } -} -``` - -**Type:** [FastSlowSpec](#fastslowspec) - -### `shard` - -Shards the data to multiple stores. This is useful for cases -when you want to distribute the load across multiple stores. -The digest hash is used to determine which store to send the -data to. - -**Example JSON5 config:** -```json5 -"shard": { - "stores": [ - { - "store": { - "memory": { - "eviction_policy": { - "max_bytes": "10mb" - }, - }, - }, - "weight": 1 - }] -} -``` - -**Type:** [ShardSpec](#shardspec) - -### `filesystem` - -Stores the data on the filesystem. This store is designed for -local persistent storage. Restarts of this program should restore -the previous state, meaning anything uploaded will be persistent -as long as the filesystem integrity holds. - -**Example JSON5 config:** -```json5 -"filesystem": { - "content_path": "/tmp/nativelink/data-worker-test/content_path-cas", - "temp_path": "/tmp/nativelink/data-worker-test/tmp_path-cas", - "eviction_policy": { - "max_bytes": "10gb", - } -} -``` - -**Type:** [FilesystemSpec](#filesystemspec) - -### `ref_store` - -Store used to reference a store in the root store manager. -This is useful for cases when you want to share a store in different -nested stores. Example, you may want to share the same memory store -used for the action cache, but use a `FastSlowSpec` and have the fast -store also share the memory store for efficiency. - -**Example JSON5 config:** -```json5 -"ref_store": { - "name": "FS_CONTENT_STORE" -} -``` - -**Type:** [RefSpec](#refspec) - -### `size_partitioning` - -Uses the size field of the digest to separate which store to send the -data. This is useful for cases when you'd like to put small objects -in one store and large objects in another store. This should only be -used if the size field is the real size of the content, in other -words, don't use on AC (Action Cache) stores. Any store where you can -safely use `VerifySpec.verify_size = true`, this store should be safe -to use (i.e., CAS stores). - -**Example JSON5 config:** -```json5 -"size_partitioning": { - "size": "128mib", - "lower_store": { - "memory": { - "eviction_policy": { - "max_bytes": "${NATIVELINK_CAS_MEMORY_CONTENT_LIMIT:-100mb}" - } - } - }, - "upper_store": { - /// This store discards data larger than 128mib. - "noop": {} - } -} -``` - -**Type:** [SizePartitioningSpec](#sizepartitioningspec) - -### `grpc` - -This store will pass-through calls to another GRPC store. This store -is not designed to be used as a sub-store of another store, but it -does satisfy the interface and will likely work. - -One major GOTCHA is that some stores use a special function on this -store to get the size of the underlying object, which is only reliable -when this store is serving the a CAS store, not an AC store. If using -this store directly without being a child of any store there are no -side effects and is the most efficient way to use it. - -**Example JSON5 config:** -```json5 -"grpc": { - "instance_name": "main", - "endpoints": [ - {"address": "grpc://${CAS_ENDPOINT:-127.0.0.1}:50051"} - ], - "connections_per_endpoint": "5", - "rpc_timeout_s": "5m", - "store_type": "ac", - // Static headers attached to every outgoing request to the upstream - // remote cache. Useful for fixed service-account credentials. - "headers": { - "authorization": "Bearer my-static-token" - }, - // Header names to copy from the inbound client request and forward to - // the upstream remote cache. Use this to pass through dynamic - // credentials such as a JWT sent by the build client. - "forward_headers": ["authorization", "x-custom-token"] -} -``` - -**Type:** [GrpcSpec](#grpcspec) - -### `redis_store` - -Stores data in any stores compatible with Redis APIs. - -Pairs well with `SizePartitioning` and/or `FastSlow` stores. -Ideal for accepting small object sizes as most Redis store -services have a max file upload of between 256Mb-512Mb. - -**Example JSON5 config:** -```json5 -"redis_store": { - "addresses": [ - "redis://127.0.0.1:6379/", - ], - "max_client_permits": 1000, -} -``` - -**Type:** [RedisSpec](#redisspec) - -### `noop` - -Noop store is a store that sends streams into the void and all data -retrieval will return 404 (`NotFound`). This can be useful for cases -where you may need to partition your data and part of your data needs -to be discarded. - -**Example JSON5 config:** -```json5 -"noop": {} -``` - -**Type:** [NoopSpec](#noopspec) - -### `experimental_mongo` - -Experimental `MongoDB` store implementation. - -This store uses `MongoDB` as a backend for storing data. It supports -both CAS (Content Addressable Storage) and scheduler data with -optional change streams for real-time updates. - -**Example JSON5 config:** -```json5 -"experimental_mongo": { - "connection_string": "mongodb://localhost:27017", - "database": "nativelink", - "cas_collection": "cas", - "key_prefix": "cas:", - "read_chunk_size": 65536, - "max_concurrent_uploads": 10, - "enable_change_streams": false, - "max_requests": "100" -} -``` - -**Type:** [ExperimentalMongoSpec](#experimentalmongospec) - -## EvictionPolicy - -Eviction policy always works on LRU (Least Recently Used). Any time an entry -is touched it updates the timestamp. Inserts and updates will execute the -eviction policy removing any expired entries and/or the oldest entries -until the store size becomes smaller than `max_bytes`. - -| Field | Type | Required | Default | Description | -| --- | --- | --- | --- | --- | -| `max_bytes` | integer (uint) | — | 0 | Maximum number of bytes before eviction takes place. Zero means never evict based on size. | -| `evict_bytes` | integer (uint) | — | 0 | When eviction starts based on hitting `max_bytes`, continue until `max_bytes - evict_bytes` is met to create a low watermark. This stops operations from thrashing when the store is close to the limit. | -| `max_seconds` | integer (uint32) | — | 0 | Maximum number of seconds for an entry to live since it was last accessed before it is evicted. Zero means never evict based on time. | -| `max_count` | integer (uint64) | — | 0 | Maximum size of the store before an eviction takes place. Zero means never evict based on count. | - -## Retry - -Retry configuration. This configuration is exponential and each iteration -a jitter as a percentage is applied of the calculated delay. For example: -```haskell -Retry{ - max_retries: 7, - delay: 0.1, - jitter: 0.5, -} -``` -will result in: - -| Attempt | Delay | -| --- | --- | -| 1 | 0 ms | -| 2 | 75 to 125 ms | -| 3 | 150 to 250 ms | -| 4 | 300 to 500 ms | -| 5 | 600 ms to 1 s | -| 6 | 1.2 to 2 s | -| 7 | 2.4 to 4 s | -| 8 | 4.8 to 8 s | - -The total delay is additive, so this example produces 9.525 to 15.875 s of total delay for a single request. - -| Field | Type | Required | Default | Description | -| --- | --- | --- | --- | --- | -| `max_retries` | integer (uint) | — | `0` | Maximum number of retries until retrying stops. Setting this to zero will always attempt 1 time, but not retry. | -| `delay` | number (float) | — | `0` | Delay in seconds for exponential back off. | -| `jitter` | number (float) | — | `0` | Amount of jitter to add as a percentage in decimal form. This will change the formula like: | -| `retry_on_errors` | array of [ErrorCode](#errorcode) | — | — | A list of error codes to retry on, if this is not set then the default error codes to retry on are used. These default codes are the most likely to be non-permanent: `Unknown`; `Cancelled`; `DeadlineExceeded`; `ResourceExhausted`; `Aborted`; `Internal`; `Unavailable`; `DataLoss` | - -## ExperimentalOntapS3Spec - -| Field | Type | Required | Default | Description | -| --- | --- | --- | --- | --- | -| `endpoint` | string | Yes | — | | -| `vserver_name` | string | Yes | — | | -| `bucket` | string | Yes | — | | -| `root_certificates` | string | — | — | | -| `key_prefix` | string | — | — | If you wish to prefix the location in the bucket. If None, no prefix will be used. | -| `retry` | [Retry](#retry) | — | — | Retry configuration to use when a network request fails. | -| `consider_expired_after_s` | integer (uint32) | — | 0 | If the number of seconds since the `last_modified` time of the object is greater than this value, the object will not be considered "existing". This allows for external tools to delete objects that have not been uploaded in a long time. If a client receives a `NotFound` the client should re-upload the object. | -| `max_retry_buffer_per_request` | integer (uint) | — | 5MB | The maximum buffer size to retain in case of a retryable error during upload. Setting this to zero will disable upload buffering; this means that in the event of a failure during upload, the entire upload will be aborted and the client will likely receive an error. | -| `multipart_max_concurrent_uploads` | integer (uint) | — | 10 | Maximum number of concurrent `UploadPart` requests per `MultipartUpload`. | -| `insecure_allow_http` | boolean | — | false | Allow unencrypted HTTP connections. Only use this for local testing. | -| `disable_http2` | boolean | — | false | Disable HTTP/2 connections and only use HTTP/1.1. Default client configuration will have HTTP/1.1 and HTTP/2 enabled for connection schemes. HTTP/2 should be disabled if environments have poor support or performance related to HTTP/2. Safe to keep default unless underlying network environment, S3, or GCS API servers specify otherwise. | - -## CompressionAlgorithm - -Plus exactly one of the following variants (the key selects the variant): - -### `lz4` - -LZ4 compression algorithm is extremely fast for compression and -decompression, however does not perform very well in compression -ratio. In most cases build artifacts are highly compressible, however -lz4 is quite good at aborting early if the data is not deemed very -compressible. - -see: [https://lz4.github.io/lz4/](https://lz4.github.io/lz4/) - -**Type:** [Lz4Config](#lz4config) - -## StoreDirection - -| Value | Description | -| --- | --- | -| `"both"` | The store operates normally and all get and put operations are handled by it. | -| `"update"` | Update operations will cause persistence to this store, but Get operations will be ignored. This only makes sense on the fast store as the slow store will never get written to on Get anyway. | -| `"get"` | Get operations will cause persistence to this store, but Update operations will be ignored. | -| `"read_only"` | Operate as a read only store, only really makes sense if there's another way to write to it. | - -## ShardConfig - -Configuration for an individual shard of the store. - -| Field | Type | Required | Default | Description | -| --- | --- | --- | --- | --- | -| `store` | [StoreSpec](#storespec) | Yes | — | Store to shard the data to. | -| `weight` | integer (uint32) | — | 1 | The weight of the store. This is used to determine how much data should be sent to the store. The actual percentage is the sum of all the store's weights divided by the individual store's weight. | - -## GrpcEndpoint - -| Field | Type | Required | Default | Description | -| --- | --- | --- | --- | --- | -| `address` | string | Yes | — | The endpoint address (i.e. grpc://example.com:443 or grpcs://example.com:443). | -| `tls_config` | [ClientTlsConfig](#clienttlsconfig) | — | — | The TLS configuration to use to connect to the endpoint (if grpcs). | -| `concurrency_limit` | integer (uint) | — | — | The maximum concurrency to allow on this endpoint. | -| `connect_timeout_s` | integer (uint64) | — | 30 seconds | Timeout for establishing a TCP connection to the endpoint (seconds). | -| `tcp_keepalive_s` | integer (uint64) | — | 30 seconds | TCP keepalive interval (seconds). Sends TCP keepalive probes at this interval to detect dead connections at the OS level. | -| `http2_keepalive_interval_s` | integer (uint64) | — | 30 seconds | HTTP/2 keepalive interval (seconds). Sends HTTP/2 PING frames at this interval to detect dead connections at the application level. | -| `http2_keepalive_timeout_s` | integer (uint64) | — | 20 seconds | HTTP/2 keepalive timeout (seconds). If a PING response is not received within this duration, the connection is considered dead. | - -## StoreType - -| Value | Description | -| --- | --- | -| `"cas"` | The store is content addressable storage. | -| `"ac"` | The store is an action cache. | +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `max_blob_size_bytes` | integer (uint64) | — | 131072 (128 KiB) | Only blobs at or below this size (in bytes) are eligible for batching. Larger blobs always use the `ByteStream` `Read` path. | +| `max_batch_bytes` | integer (uint64) | — | 3145728 (3 MiB) | Maximum total payload bytes packed into a single `BatchReadBlobs` request. This should leave headroom under the 4 MiB default gRPC message limit for protobuf framing overhead. | +| `dispatch_slots` | integer (uint) | — | 4 | Maximum number of concurrent `BatchReadBlobs` RPCs dispatched by the coalescer. Must be greater than zero. | +| `max_queued_bytes` | integer (uint64) | — | 33554432 (32 MiB) | Bound on the number of payload bytes waiting in the coalescer queue. When exceeded, new read requests bypass batching and fall back to the regular `ByteStream` `Read` path instead of blocking. | ## RedisMode @@ -1733,6 +1224,9 @@ One of: | `max_entries` | integer (uint) | — | 1000 | Maximum number of cached directories. | | `max_size_bytes` | integer (uint64) | — | 10737418240 (10 GB) | Maximum total size in bytes for all cached directories (0 = unlimited). | | `cache_root` | string | — | `{work_directory}/../directory_cache` | Base directory for cache storage. This directory will be managed by the worker and should be on the same filesystem as `work_directory`. | +| `experimental_subtree_caching` | boolean | — | false (only root directories are cached; existing behavior) | Optional and experimental: additionally cache every subdirectory by its own `Directory` digest, not just the root directory. REAPI Merkle nodes are content-addressed, so a subtree that is byte-identical between two different roots has the same digest and can be materialized with a single hardlink pass instead of being rebuilt from the CAS. This makes the common "one input file changed out of thousands" case reuse every unchanged subtree. | +| `max_concurrent_fetches` | integer (uint) | — | 64 | Maximum number of concurrent slow-store fetches across ALL directory constructions of this cache. This bound protects backing stores from RPC storms: per-level construction concurrency compounds multiplicatively across tree levels and concurrent actions. | +| `experimental_get_tree_prefetch` | boolean | — | false | On a directory-cache miss, prefetch every `Directory` proto of the tree with one logical `GetTree` traversal instead of fetching protos level by level (which costs one round trip per tree DEPTH). The [REAPI request contract] permits servers to impose their own limit even when no page size is specified, and the [REAPI response contract] requires clients to continue with the returned page token. A paginated traversal may therefore use multiple RPCs. Only takes effect when the slow tier is a `grpc` store; any prefetch failure falls back to the per-level path. Worthwhile when worker-to-CAS latency is non-trivial and trees are deep; measured 5-34x on the proto phase at 5-25ms RTT. Note the serving CAS pays the tree walk against its own backend, so its directory protos should be served from a fast tier. | ## PropertyType @@ -1827,6 +1321,7 @@ If a property is found, then replace it with another one. | --- | --- | --- | --- | --- | | `instance_name` | string | — | `""` | Used when the config references `instance_name` in the protocol. | | `cas_store` | string | Yes | — | The store name referenced in the `stores` map in the main config. This store name referenced here may be reused multiple times. | +| `experimental_chunking` | [CasChunkingConfig](#caschunkingconfig) | — | not set — chunking RPCs are rejected, nothing is advertised, | Optional and experimental: enables the REAPI `SplitBlob`/`SpliceBlob` RPCs used by content-defined chunking clients (e.g. Bazel's `--experimental_remote_cache_chunking`). When set, the capabilities service advertises blob split/splice support and `FastCDC` 2020 parameters for this instance. When `cas_store` is a grpc store the RPCs are forwarded to the backend (which must support chunking with matching parameters); otherwise they are served locally. | ## ActionCacheServiceConfig @@ -1842,6 +1337,7 @@ If a property is found, then replace it with another one. | --- | --- | --- | --- | --- | | `instance_name` | string | — | `""` | Used when the config references `instance_name` in the protocol. | | `remote_execution` | [CapabilitiesRemoteExecutionConfig](#capabilitiesremoteexecutionconfig) | — | — | Configuration for remote execution capabilities. If not set the capabilities service will inform the client that remote execution is not supported. | +| `remote_cache_compression` | boolean | — | — | Whether this instance supports Bazel remote cache compression. When enabled, the capabilities service advertises zstd wire compression and the ByteStream/CAS services accept REAPI compressed-blobs/zstd data. | ## ExecutionServiceConfig @@ -1858,7 +1354,7 @@ If a property is found, then replace it with another one. | `instance_name` | string | — | `""` | Used when the config references `instance_name` in the protocol. | | `cas_store` | string | Yes | — | Name of the store in the "stores" configuration. | | `max_bytes_per_stream` | integer (uint) | — | 64KiB | Max number of bytes to send on each grpc stream chunk. According to [https://github.com/grpc/grpc.github.io/issues/371](https://github.com/grpc/grpc.github.io/issues/371) 16KiB - 64KiB is optimal. | -| `persist_stream_on_disconnect_timeout` | integer (uint) | — | 10 seconds | In the event a client disconnects while uploading a blob, we will hold the internal stream open for this many seconds before closing it. This allows clients that disconnect to reconnect and continue uploading the same blob. | +| `persist_stream_on_disconnect_timeout_s` | integer (uint) | — | 10 seconds | In the event a client disconnects while uploading a blob, we will hold the internal stream open for this many seconds before closing it. This allows clients that disconnect to reconnect and continue uploading the same blob. | ## FetchServiceConfig @@ -2005,7 +1501,7 @@ specified. | `experimental_http2_adaptive_window` | boolean | — | — | | | `experimental_http2_max_frame_size` | integer (uint32) | — | — | | | `experimental_http2_max_concurrent_streams` | integer (uint32) | — | — | | -| `experimental_http2_keep_alive_timeout` | integer (uint32) | — | — | Note: This is in seconds. | +| `experimental_http2_keep_alive_timeout_s` | integer (uint32) | — | — | | | `experimental_http2_max_send_buf_size` | integer (uint32) | — | — | | | `experimental_http2_enable_connect_protocol` | boolean | — | — | | | `experimental_http2_max_header_list_size` | integer (uint32) | — | — | | @@ -2019,6 +1515,14 @@ specified. | `client_ca_file` | string | — | — | Path to the certificate authority for mTLS, if client authentication is required for this endpoint. | | `client_crl_file` | string | — | — | Path to the certificate revocation list for mTLS, if client authentication is required for this endpoint. | +## CasChunkingConfig + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `index_store` | string | — | — | The store name referenced in the `stores` map in the main config used to persist blob-to-chunks layouts. Keys are the digests of the original blobs and values are serialized chunk layouts (which do not hash to those digests), so this store MUST NOT perform content digest verification and MUST NOT be the same store as `cas_store` — writing layouts into the CAS would overwrite blob content. Using the same store name as `cas_store` is rejected at startup. | +| `avg_chunk_size_bytes` | integer (uint64) | — | 524288 (512 KiB) | The average chunk size in bytes advertised to clients through the `FastCDC` 2020 capability parameters and used for server-side chunking in `SplitBlob`. Clients derive the minimum and maximum chunk sizes from this value (avg / 4 and avg * 4). The value must be between 1 KiB and 1 MiB. | +| `max_chunk_count` | integer (uint64) | — | 50000 | Maximum number of chunks accepted in a `SpliceBlob` request or produced by on-demand chunking in `SplitBlob`. Blobs that would produce more chunks are served without chunking (`SplitBlob` returns `NOT_FOUND` and clients fall back to a regular download). This bounds the size of stored chunk layouts and of `SplitBlobResponse` messages (roughly 80-140 bytes per chunk). At the default average chunk size the default cap supports blobs up to ~25 GiB; note that values above ~50000 may produce responses that exceed default gRPC message size limits on clients. | + ## CapabilitiesRemoteExecutionConfig | Field | Type | Required | Default | Description | @@ -2036,6 +1540,6 @@ specified. If anything here disagrees with the binary, the source wins: -- [`stores.rs`](https://github.com/TraceMachina/nativelink/tree/v1.5.2/nativelink-config/src/stores.rs) -- [`cas_server.rs`](https://github.com/TraceMachina/nativelink/tree/v1.5.2/nativelink-config/src/cas_server.rs) -- [`schedulers.rs`](https://github.com/TraceMachina/nativelink/tree/v1.5.2/nativelink-config/src/schedulers.rs) +- [`stores.rs`](https://github.com/TraceMachina/nativelink/tree/v1.6.3/nativelink-config/src/stores.rs) +- [`cas_server.rs`](https://github.com/TraceMachina/nativelink/tree/v1.6.3/nativelink-config/src/cas_server.rs) +- [`schedulers.rs`](https://github.com/TraceMachina/nativelink/tree/v1.6.3/nativelink-config/src/schedulers.rs) diff --git a/web/apps/docs/content/docs/reference/nativelink-config/main.mdx b/web/apps/docs/content/docs/reference/nativelink-config/main.mdx index 3ceba63c6..798ef5b69 100644 --- a/web/apps/docs/content/docs/reference/nativelink-config/main.mdx +++ b/web/apps/docs/content/docs/reference/nativelink-config/main.mdx @@ -5,7 +5,7 @@ full: true --- {/* AUTOGENERATED — do not edit by hand. - Source: nativelink-config @ main (1a28c5fe) + Source: nativelink-config @ main (a83dbc1a) Regenerate from web/: bun --filter @nativelink/docs gen:config-reference */} @@ -151,8 +151,8 @@ It supports the following backends: } ``` -4. **`NetApp` ONTAP S3** - `NetApp` ONTAP S3 store will use ONTAP's S3-compatible storage as a backend +4. **NetApp ONTAP S3:** + NetApp ONTAP S3 store will use ONTAP's S3-compatible storage as a backend to store files. This store is specifically configured for ONTAP's S3 requirements including custom TLS configuration, credentials management, and proper vserver configuration. @@ -451,7 +451,7 @@ On uploads it will mirror data to both `fast` and `slow` stores. WARNING: If you need data to always exist in the `slow` store for something like remote execution, be careful because this store will never check to see if the objects exist in the -`slow` store if it exists in the `fast` store (i.e., it assumes +`slow` store if it exists in the `fast` store (i.e. it assumes that if an object exists in the `fast` store it will exist in the `slow` store). @@ -552,7 +552,7 @@ in one store and large objects in another store. This should only be used if the size field is the real size of the content, in other words, don't use on AC (Action Cache) stores. Any store where you can safely use `VerifySpec.verify_size = true`, this store should be safe -to use (i.e., CAS stores). +to use (i.e. CAS stores). **Example JSON5 config:** ```json5 @@ -576,7 +576,7 @@ to use (i.e., CAS stores). ### `grpc` -This store will pass-through calls to another GRPC store. This store +This store will pass-through calls to another gRPC store. This store is not designed to be used as a sub-store of another store, but it does satisfy the interface and will likely work. @@ -618,6 +618,12 @@ Pairs well with `SizePartitioning` and/or `FastSlow` stores. Ideal for accepting small object sizes as most Redis store services have a max file upload of between 256Mb-512Mb. +If you are using Redis together with any stores above it +e.g. existence cache, you will need to configure `notify-keyspace-events` +to `KA` as per [https://redis.io/docs/latest/develop/pubsub/keyspace-notifications/#configuration](https://redis.io/docs/latest/develop/pubsub/keyspace-notifications/#configuration) +in order for us to get eviction events. Failing to do so will get you +log messages complaining about it, as well as errors like [https://github.com/TraceMachina/nativelink/issues/2436](https://github.com/TraceMachina/nativelink/issues/2436) + **Example JSON5 config:** ```json5 "redis_store": { @@ -817,12 +823,13 @@ See [`experimental_cloud_object_store`](#experimental_cloud_object_store-1) for | Field | Type | Required | Default | Description | | --- | --- | --- | --- | --- | -| `content_path` | string | Yes | — | Path on the system where to store the actual content. This is where the bulk of the data will be placed. On service startup this folder will be scanned and all files will be added to the cache. In the event one of the files doesn't match the criteria, the file will be deleted. | -| `temp_path` | string | Yes | — | A temporary location of where files that are being uploaded or deleted will be placed while the content cannot be guaranteed to be accurate. This location must be on the same block device as `content_path` so atomic moves can happen (i.e., move without copy). All files in this folder will be deleted on every startup. | +| `content_path` | string | Yes | — | Path on the system where to store the actual content. This is where the bulk of the data will be placed. On service boot this folder will be scanned and all files will be added to the cache. In the event one of the files doesn't match the criteria, the file will be deleted. | +| `temp_path` | string | Yes | — | A temporary location of where files that are being uploaded or deleted will be placed while the content cannot be guaranteed to be accurate. This location must be on the same block device as `content_path` so atomic moves can happen (i.e. move without copy). All files in this folder will be deleted on every startup. | | `read_buffer_size` | integer (uint32) | — | 32k | Buffer size to use when reading files. Generally this should be left to the default value except for testing. | | `eviction_policy` | [EvictionPolicy](#evictionpolicy) | — | — | Policy used to evict items out of the store. Failure to set this value will cause items to never be removed from the store causing infinite memory usage. | | `block_size` | integer (uint64) | — | 4kb | The block size of the filesystem for the running machine value is used to determine an entry's actual size on disk consumed For a 4KB block size filesystem, a 1B file actually consumes 4KB | | `max_concurrent_writes` | integer (uint) | — | unlimited | Maximum number of concurrent write operations allowed. Each write involves streaming data to a temp file and calling `sync_all()`, which can saturate disk I/O when many writes happen simultaneously. Limiting concurrency prevents disk saturation from blocking the async runtime. A value of 0 means unlimited (no concurrency limit). | +| `evict_page_cache` | boolean | — | false | When true, advise the kernel to drop the page cache for each blob after it is written or read (`posix_fadvise` with `POSIX_FADV_DONTNEED`). On real filesystems this takes a globally serialized, all-CPU kernel path that stalls on many-core hosts, and it evicts the page-cache tier that fast-disk deployments rely on. Leave off unless you specifically want to keep this store's I/O out of the page cache. | ## RefSpec @@ -842,16 +849,17 @@ See [`experimental_cloud_object_store`](#experimental_cloud_object_store-1) for | Field | Type | Required | Default | Description | | --- | --- | --- | --- | --- | -| `instance_name` | string | — | `""` | Instance name for GRPC calls. Proxy calls will have the `instance_name` changed to this. | +| `instance_name` | string | — | `""` | Instance name for gRPC calls. Proxy calls will have the `instance_name` changed to this. | | `endpoints` | array of [GrpcEndpoint](#grpcendpoint) | Yes | — | The endpoint of the grpc connection. | | `store_type` | [StoreType](#storetype) | Yes | — | The type of the upstream store, this ensures that the correct server calls are made. | | `retry` | [Retry](#retry) | — | — | Retry configuration to use when a network request fails. | | `max_concurrent_requests` | integer (uint) | — | `0` | Limit the number of simultaneous upstream requests to this many. A value of zero is treated as unlimited. If the limit is reached the request is queued. | -| `connections_per_endpoint` | integer (uint) | — | `0` | The number of connections to make to each specified endpoint to balance the load over multiple TCP connections. Default 1. | +| `connections_per_endpoint` | integer (uint) | — | 1 | The number of connections to make to each specified endpoint to balance the load over multiple TCP connections. | | `rpc_timeout_s` | integer (uint64) | — | 0 (disabled) | Maximum time (seconds) allowed for a single RPC request (e.g. a `ByteStream.Write` call) before it is cancelled. | | `use_legacy_resource_names` | boolean | — | false | Use legacy `ByteStream` resource name format, omitting the digest function component from the path. | | `headers` | map of string to string | — | — | Static headers to attach to every outgoing gRPC request sent to this store's upstream endpoints. Useful for fixed authentication tokens (e.g. `{"authorization": "Bearer "}`) and other static metadata. | | `forward_headers` | array of string | — | — | Header names to forward from the incoming client request to every outgoing upstream request. The header value is taken from the client request that triggered this store operation. Use this to pass through dynamic credentials such as JWT tokens sent by build clients. | +| `experimental_read_batching` | [GrpcReadBatchingConfig](#grpcreadbatchingconfig) | — | unset (disabled) | Optional and experimental: coalesce small-blob reads into `BatchReadBlobs` RPCs instead of issuing one `ByteStream` `Read` stream per blob. Each `ByteStream` read carries a fixed per-RPC cost, so batching many small reads into a single `BatchReadBlobs` request can dramatically reduce read latency for small blobs. | ## RedisSpec @@ -869,7 +877,7 @@ See [`experimental_cloud_object_store`](#experimental_cloud_object_store-1) for | `health_check_timeout_ms` | integer (uint64) | — | 4000 (4 seconds) | Per-call ceiling for the `check_health` PING in milliseconds. | | `read_chunk_size` | integer (uint) | — | 64KiB | The amount of data to read from the Redis server at a time. This is used to limit the amount of memory used when reading large objects from the Redis server as well as limiting the amount of time a single read operation can take. | | `connection_pool_size` | integer (uint) | — | 3 | The number of connections to keep open to the Redis servers. | -| `max_chunk_uploads_per_update` | integer (uint) | — | 10 | The maximum number of upload chunks to allow per update. This is used to limit the amount of memory used when uploading large objects to the Redis server. A good rule of thumb is to think of the data as: `AVAIL_MEMORY / (read_chunk_size * max_chunk_uploads_per_update) = THORETICAL_MAX_CONCURRENT_UPLOADS` (note: it is a good idea to divide `AVAIL_MAX_MEMORY` by ~10 to account for other memory usage) | +| `max_chunk_uploads_per_update` | integer (uint) | — | 10 | The maximum number of upload chunks to allow per update. This is used to limit the amount of memory used when uploading large objects to the Redis server. A good rule of thumb is to think of the data as: `AVAIL_MEMORY / (read_chunk_size * max_chunk_uploads_per_update) = THORETICAL_MAX_CONCURRENT_UPLOADS` (note: it's a good idea to divide `AVAIL_MAX_MEMORY` by ~10 to account for other memory usage) | | `scan_count` | integer (uint) | — | 10000 | The COUNT value passed when scanning keys in Redis. This is used to hint the amount of work that should be done per response. | | `retry` | [Retry](#retry) | — | — | Retry configuration to use when a network request fails. | | `max_client_permits` | integer (uint) | — | 500 | Maximum number of permitted actions to the Redis store at any one time This stops problems with timeouts due to many, many inflight actions | @@ -904,7 +912,7 @@ Configuration for `ExperimentalMongoDB` store. | Field | Type | Required | Default | Description | | --- | --- | --- | --- | --- | -| `name` | string | — | {Index position in the workers list} | Name of the worker. This is give a more friendly name to a worker for logging and metric publishing. This is also the prefix of the worker id (i.e., "{name}{uuidv6}"). | +| `name` | string | — | {Index position in the workers list} | Name of the worker. This is give a more friendly name to a worker for logging and metric publishing. This is also the prefix of the worker id (i.e. "{name}{uuidv6}"). | | `worker_api_endpoint` | [EndpointConfig](#endpointconfig) | Yes | — | Endpoint which the worker will connect to the scheduler's `WorkerApiService`. | | `max_action_timeout_s` | integer (uint) | — | 20 minutes | The maximum time an action is allowed to run. If a task requests for a timeout longer than this time limit, the task will be rejected. Value in seconds. | | `max_upload_timeout_s` | integer (uint) | — | 10 minutes | Maximum time allowed for uploading action results to CAS after execution completes. If upload takes longer than this, the action fails with `DeadlineExceeded` and may be retried by the scheduler. Value in seconds. | @@ -936,6 +944,7 @@ Configuration for `ExperimentalMongoDB` store. | `allocation_strategy` | [WorkerAllocationStrategy](#workerallocationstrategy) | — | `"least_recently_used"` | The strategy used to assign workers jobs. | | `experimental_backend` | [ExperimentalSimpleSchedulerBackend](#experimentalsimpleschedulerbackend) | — | memory | The storage backend to use for the scheduler. | | `worker_match_logging_interval_s` | integer (int64) | — | `10` | Every N seconds, do logging of worker matching e.g. "worker busy", "can't find any worker" Defaults to 10s. Can be set to `-1` to disable | +| `fallback_match_interval_s` | integer (int64) | — | `5` | Every N seconds, run a worker matching pass even if no task or worker change notification arrived. This is a safety net for missed notifications and for scheduler backends with eventually consistent searches (for example Redis), where an operation that was re-queued may not be visible to the search triggered by its own notification. Without this, such an operation can stay queued until an unrelated event triggers another matching pass. Defaults to 5s. Zero or any negative value disables it. | ## SchedulerGrpcSpec @@ -991,7 +1000,7 @@ Listener for HTTP/HTTPS/HTTP2 sockets. | --- | --- | --- | --- | --- | | `cas` | array of [CasServiceConfig](#casserviceconfig) | — | — | The Content Addressable Storage (CAS) backend config. The key is the `instance_name` used in the protocol and the value is the underlying CAS store config. | | `ac` | array of [ActionCacheServiceConfig](#actioncacheserviceconfig) | — | — | The Action Cache (AC) backend config. The key is the `instance_name` used in the protocol and the value is the underlying AC store config. | -| `capabilities` | array of [CapabilitiesServiceConfig](#capabilitiesserviceconfig) | — | — | Capabilities service is required in order to use most of the bazel protocol. This service is used to provide the supported features and versions of this bazel GRPC service. | +| `capabilities` | array of [CapabilitiesServiceConfig](#capabilitiesserviceconfig) | — | — | Capabilities service is required in order to use most of the bazel protocol. This service is used to provide the supported features and versions of this bazel gRPC service. | | `execution` | array of [ExecutionServiceConfig](#executionserviceconfig) | — | — | The remote execution service configuration. NOTE: This service is under development and is currently just a place holder. | | `bytestream` | array of [ByteStreamServiceConfig](#bytestreamserviceconfig) | — | — | This is the service used to stream data to and from the CAS. Bazel's protocol strongly encourages users to use this streaming interface to interact with the CAS when the data is large. | | `fetch` | array of [FetchServiceConfig](#fetchserviceconfig) | — | — | These two are collectively the Remote Asset protocol, but it's defined as two separate services | @@ -1066,7 +1075,7 @@ The total delay is additive, so this example produces 9.525 to 15.875 s of total | `max_retries` | integer (uint) | — | `0` | Maximum number of retries until retrying stops. Setting this to zero will always attempt 1 time, but not retry. | | `delay` | number (float) | — | `0` | Delay in seconds for exponential back off. | | `jitter` | number (float) | — | `0` | Amount of jitter to add as a percentage in decimal form. This will change the formula like: | -| `retry_on_errors` | array of [ErrorCode](#errorcode) | — | — | A list of error codes to retry on, if this is not set then the default error codes to retry on are used. These default codes are the most likely to be non-permanent: `Unknown`; `Cancelled`; `DeadlineExceeded`; `ResourceExhausted`; `Aborted`; `Internal`; `Unavailable`; `DataLoss` | +| `retry_on_errors` | array of [ErrorCode](#errorcode) | — | — | A list of error codes to retry on, if this isn't set then the default error codes to retry on are used. These default codes are the most likely to be non-permanent: `Unknown`; `Cancelled`; `DeadlineExceeded`; `ResourceExhausted`; `Aborted`; `Internal`; `Unavailable`; `DataLoss` | ## ExperimentalOntapS3Spec @@ -1122,7 +1131,7 @@ Configuration for an individual shard of the store. | Field | Type | Required | Default | Description | | --- | --- | --- | --- | --- | -| `address` | string | Yes | — | The endpoint address (i.e. grpc://example.com:443 or grpcs://example.com:443). | +| `address` | string | Yes | — | The endpoint address (i.e. `grpc(s)://example.com:443`). | | `tls_config` | [ClientTlsConfig](#clienttlsconfig) | — | — | The TLS configuration to use to connect to the endpoint (if grpcs). | | `concurrency_limit` | integer (uint) | — | — | The maximum concurrency to allow on this endpoint. | | `connect_timeout_s` | integer (uint64) | — | 30 seconds | Timeout for establishing a TCP connection to the endpoint (seconds). | @@ -1137,6 +1146,18 @@ Configuration for an individual shard of the store. | `"cas"` | The store is content addressable storage. | | `"ac"` | The store is an action cache. | +## GrpcReadBatchingConfig + +Configuration for experimental small-blob read coalescing in a gRPC +store. See [`GrpcSpec::experimental_read_batching`]. + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `max_blob_size_bytes` | integer (uint64) | — | 131072 (128 KiB) | Only blobs at or below this size (in bytes) are eligible for batching. Larger blobs always use the `ByteStream` `Read` path. | +| `max_batch_bytes` | integer (uint64) | — | 3145728 (3 MiB) | Maximum total payload bytes packed into a single `BatchReadBlobs` request. This should leave headroom under the 4 MiB default gRPC message limit for protobuf framing overhead. | +| `dispatch_slots` | integer (uint) | — | 4 | Maximum number of concurrent `BatchReadBlobs` RPCs dispatched by the coalescer. Must be greater than zero. | +| `max_queued_bytes` | integer (uint64) | — | 33554432 (32 MiB) | Bound on the number of payload bytes waiting in the coalescer queue. When exceeded, new read requests bypass batching and fall back to the regular `ByteStream` `Read` path instead of blocking. | + ## RedisMode | Value | Description | @@ -1203,6 +1224,9 @@ One of: | `max_entries` | integer (uint) | — | 1000 | Maximum number of cached directories. | | `max_size_bytes` | integer (uint64) | — | 10737418240 (10 GB) | Maximum total size in bytes for all cached directories (0 = unlimited). | | `cache_root` | string | — | `{work_directory}/../directory_cache` | Base directory for cache storage. This directory will be managed by the worker and should be on the same filesystem as `work_directory`. | +| `experimental_subtree_caching` | boolean | — | false (only root directories are cached; existing behavior) | Optional and experimental: additionally cache every subdirectory by its own `Directory` digest, not just the root directory. REAPI Merkle nodes are content-addressed, so a subtree that is byte-identical between two different roots has the same digest and can be materialized with a single hardlink pass instead of being rebuilt from the CAS. This makes the common "one input file changed out of thousands" case reuse every unchanged subtree. | +| `max_concurrent_fetches` | integer (uint) | — | 64 | Maximum number of concurrent slow-store fetches across ALL directory constructions of this cache. This bound protects backing stores from RPC storms: per-level construction concurrency compounds multiplicatively across tree levels and concurrent actions. | +| `experimental_get_tree_prefetch` | boolean | — | false | On a directory-cache miss, prefetch every `Directory` proto of the tree with one logical `GetTree` traversal instead of fetching protos level by level (which costs one round trip per tree DEPTH). The [REAPI request contract] permits servers to impose their own limit even when no page size is specified, and the [REAPI response contract] requires clients to continue with the returned page token. A paginated traversal may therefore use multiple RPCs. Only takes effect when the slow tier is a `grpc` store; any prefetch failure falls back to the per-level path. Worthwhile when worker-to-CAS latency is non-trivial and trees are deep; measured 5-34x on the proto phase at 5-25ms RTT. Note the serving CAS pays the tree walk against its own backend, so its directory protos should be served from a fast tier. | ## PropertyType diff --git a/web/apps/docs/content/docs/reference/nativelink-config/store-overview.mdx b/web/apps/docs/content/docs/reference/nativelink-config/store-overview.mdx index ca4b343fa..5e6ccfab4 100644 --- a/web/apps/docs/content/docs/reference/nativelink-config/store-overview.mdx +++ b/web/apps/docs/content/docs/reference/nativelink-config/store-overview.mdx @@ -69,7 +69,7 @@ store name the server exposes. | Store key | Wraps | What it adds | | --- | --- | --- | -| `compression` | one store | LZ4-compresses on write, decompresses on read. | +| `compression` | one store | Select `lz4` for NativeLink's LZ4 framing, or `zstd` for standard zstd frames and optional wire-compression passthrough. | | `dedup` | two stores (`index_store` + `content_store`) | Rolling-hash chunking so only changed slices upload. | | `fast_slow` | two stores (`fast` + `slow`) | Reads try `fast` first, fall back to `slow`, and backfill `fast`. Writes mirror to both. | | `shard` | N stores | Routes by digest hash. The standard shape for scaling CAS past one backend. | @@ -91,8 +91,234 @@ store name the server exposes. - `existence_cache` and `size_partitioning` are CAS-only; `completeness_checking` is AC-only. Using them on the other store type produces confusing correctness bugs, not a config error. + - `compression` with `compression_algorithm: { zstd: ... }` is the one + algorithm choice that isn't position-agnostic: it must be the store an + instance points at directly, or its main benefit doesn't apply. See the + dedicated section below before deploying it. +## Pass-through compression: `compression_algorithm.zstd` + +The `zstd` algorithm of `compression` keeps CAS blobs as zstd streams **at +rest** instead of raw bytes. Unlike `compression_algorithm: { lz4: ... }` +(which uses NativeLink's LZ4 framing and decompresses back to raw bytes at +the `StoreDriver` boundary), `compression_algorithm: { zstd: ... }` can serve its stored bytes +**byte-for-byte** to a Bazel client running `--remote_cache_compression` on +a cache hit — no decompress-then-recompress round trip at the gRPC +boundary. Identity clients (no `--remote_cache_compression`) are unaffected: +they still receive plain decompressed bytes. See +[Remote cache compression](/configuration/compression) for the wire-level +feature this pairs with. + +It is **CAS-only** — every operation requires a digest key. Never point an +AC store at it. + +**Example JSON5 config:** + +```json5 +{ + name: "CAS_MAIN_STORE", + compression: { + compression_algorithm: { + zstd: { + temp_path: "/var/tmp/nativelink-zstd", + max_compressed_upload_size: "512MiB", + max_concurrent_staged_uploads: 4, + max_concurrent_identity_ops: 256, + compression_level: 9, + max_recompression_size: "64MiB", + max_concurrent_recompressions: 1, + max_inline_commit_size: "4MiB", + stage_timeout_s: 600, + commit_timeout_s: 300, + }, + }, + backend: { memory: { eviction_policy: { max_bytes: "10GiB" } } }, + }, +} +``` + +### Placement: it must be the outermost store + +Every wrapper in the table above can nest at any depth. `compression` with +`compression_algorithm.zstd` is different: **it must be the store the +instance/CAS/ByteStream service points at directly** for the byte-for-byte +passthrough to apply. `fast_slow`, +`dedup`, `existence_cache`, `cache_metrics`, `shard`, `ref_store`, and +`size_partitioning` are all fine **inside** it — they then operate on the +physical zstd stream exactly as they would on raw bytes. + +A wrapper placed **outside** the zstd compression store (for example, +`verify` wrapping it) is still correct, but that wrapper has to decode the +stream to do its job, which disables passthrough at that boundary. + + + The `backend` of `compression_algorithm.zstd` stores digest → zstd bytes, + not digest → raw bytes. The digest key is identical either way, and raw CAS payloads are + unstructured — an arbitrary raw blob can legitimately start with the zstd + magic number — so there is no safe way to sniff which encoding a given + entry is in. Version 1 has no in-place migration. + + - `backend` must point at a brand-new or empty store namespace, never one + a raw-bytes reader or writer touches — including a plain + `filesystem`/`experimental_cloud_object_store` CAS, a different + instance, or a pre-zstd NativeLink version pointed at the same + path/bucket/prefix. + - Rollout **and** rollback both require a cache flush (or cutting over to + a new namespace) — there is no supported way to select or remove zstd + compression in place against a populated namespace. + - A rolling/mixed-version deployment against **one shared namespace** is + unsupported: while some replicas write raw bytes and others write zstd + to the same keys, every reader misinterprets roughly half the entries. + + +### `compression_level`: capped at 19, not 22 + +`compression_level` is optional and sets the level used for **two** things: the +uploads this store compresses itself (identity clients), and the re-encoding of +incoming compressed uploads when `max_recompression_size > 0`. Omitted, it uses +level 3 and never re-encodes an already-compressed upload. When set, it must be +in `1..=19` — NativeLink rejects anything outside that range at +config-validation time (startup). + +Level 9 is a reasonable general-purpose choice; it costs a fraction of level +19's CPU per byte. Reach for the high end only when blobs are read far more often +than written. + +The zstd level isn't recorded in the frame; decoders are level-agnostic. The +actual cross-client constraint is **window size**. Standard levels 1–19 cap +the encoded frame at `windowLog ≤ 23` (≤ 8 MiB), decodable by every +Bazel/zstd-jni client using plain `libzstd` defaults. Bazel's decoder never +calls `setLongMax`, so its hard ceiling is `windowLog ≤ 27` (128 MiB) — +level 19 keeps a wide margin under that ceiling rather than chasing the +absolute cap. Zstd compression never enables long-distance matching or external +dictionaries (REAPI has no dictionary channel, and long-distance matching +alone can push `windowLog` past 23). + +### Bounds and admission: stopping a zstd bomb + +Every compressed input is bounded from both ends. `max_compressed_upload_size` +caps the compressed bytes a client may push; exceeding it fails the upload with +`RESOURCE_EXHAUSTED`. Independently, the decoder's output sink is capped at the +digest's declared uncompressed size and rejects with `INVALID_ARGUMENT` the +moment decoded bytes *would* exceed it — so a small "zstd bomb" that expands to +a huge buffer is stopped at the first over-limit block rather than being fully +materialized. Zero-digest validation streams through the same bounded decoder +with an output cap of zero: it never allocates the decoded output, and it takes +an admission permit like any other staged upload, so it cannot spawn unbounded +decode jobs. + +`max_concurrent_staged_uploads` admits a bounded number of compressed uploads at +a time, but admission alone can't stop a stalled client or backend from parking +on a slot forever. Three deadlines close that gap, and each one ends the same +way: `DEADLINE_EXCEEDED`, the staged file removed, the staging slot released. +Cancellation, any timeout, validation failure, and success all clean up. + +- **`stage_timeout_s`** — total seconds one upload may spend being validated and + staged, measured from the moment it takes a staging slot. This is the bound on + a client that *trickles* bytes: unlike an idle timeout, continuous slow + progress does not reset it. Size it against `max_compressed_upload_size` and + the slowest upload bandwidth worth serving. `0` uses the default of 600. +- **`commit_timeout_s`** — seconds to wait, after validation and staging, for + optional recompression and the inner-store commit. Bounds a stalled backend. + `0` uses the default of 300. +- **`compressed_upload_idle_timeout_s`** — a ByteStream instance setting, not a + store setting: how long to wait for the *next* `WriteRequest` of a compressed + upload. A client making continuous progress is never timed out, only an idle + one. `0` uses the default of 60. (Earlier builds reused + `persist_stream_on_disconnect_timeout_s` for this; it is now its own knob, + because how long a disconnected upload stays resumable has nothing to do with + how long an attached client may stall.) + +Two admission bounds sit alongside those deadlines: + +- **`max_concurrent_staged_uploads`** — compressed uploads validating or staging + at once. `0` uses the default of 4. +- **`max_concurrent_identity_ops`** — uncompressed (identity) reads and writes at + once. Each one holds a blocking thread for the whole transfer, so without a + bound a flood of slow identity clients can starve the process-wide blocking + pool that filesystem I/O also uses. Raising it trades that risk for more + queuing on this store; `0` uses the default of 256. + +Small compressed uploads skip staging altogether. **`max_inline_commit_size`** +(default 4 MiB) is the ceiling below which an upload is validated and committed +straight from memory — no staging file, no `fsync`. `BatchUpdateBlobs` payloads +are small and numerous, so a per-blob disk round trip would dominate their cost. +Inline uploads are validated exactly as staged ones are and take the same +staging admission permit and `stage_timeout_s`. + +### `temp_path`: staging directory, size and place it deliberately + +Zstd compression stages every compressed upload larger than +`max_inline_commit_size` as a temp file under `temp_path` before committing it to +`backend`. Recompression keeps the decoded bytes in bounded memory and overwrites +the same validated descriptor only when the result is smaller, so it never adds a +second staging file. + +`temp_path` must be a private directory owned by the operator and **not** writable +by local users you do not trust. At startup the store verifies it is a directory, rejects +a world-writable directory that lacks the sticky bit, and probes that it is +writable. Each staging file is created exclusively (`O_EXCL` / `create_new`) under +an unguessable name, with `0o600` applied atomically at creation rather than by a +later `chmod`. Every staged file is removed on success, error, and cancellation. + +- **What defends against an observe-and-replace race depends on the backend.** + Both the validated descriptor and its pathname are handed to `backend` at + commit. Backends that stream the bytes — the default, so `memory`, + `experimental_cloud_object_store`, and friends — read the exact descriptor that + was validated, and never reopen the path. A `filesystem` backend instead drops + the descriptor and commits with `rename(2)` on the pathname, so for that backend + the guarantee rests on exclusive creation of an unguessable name inside an + operator-private directory, which is why the directory checks above are not + optional. +- **Worst-case temp disk usage** is roughly `max_concurrent_staged_uploads + * max_compressed_upload_size`. Recompression uses bounded memory, capped by + `max_recompression_size`, rather than an additional staging file. Size and + monitor `temp_path` against the disk bound, not just steady-state usage. +- **Put `temp_path` on the same filesystem as the `content_path` of a + `filesystem` backend.** That backend commits with the `rename(2)` above, and a + cross-filesystem rename fails with `EXDEV` — the upload fails, the blob is + never stored, and the temp file is still cleaned up — instead of silently + falling back to a copy. + +### Recompression: keep whichever encoding is smaller + +With `compression_level` set and `max_recompression_size > 0`, an incoming +upload whose *decoded* size is within `max_recompression_size` gets +re-encoded at `compression_level`, and zstd compression keeps whichever of the +original or re-encoded stream is smaller. `max_recompression_size: 0` +disables recompression entirely — `compression_level` still picks the level +used to encode brand-new, not-already-compressed uploads; incoming compressed +streams are left unchanged. + +Because re-encoding needs a level, `max_recompression_size > 0` **requires** +`compression_level`. Setting the ceiling without a level is rejected at startup +rather than silently disabling recompression. + +`max_concurrent_recompressions` bounds how many re-encodes run at once, and it is +**best-effort**: an upload that finds every slot busy commits its original stream +instead of queuing. That matters because the upload is holding a staging slot +while it waits, so a small recompression pool would otherwise throttle the entire +upload path. Skipped recompressions are counted in the store's metrics +(`recompressions_skipped_busy`); a persistently high rate means the pool is too +small for the offered load, not that anything failed. Recompression also runs +under the same `commit_timeout_s` deadline as the inner-store commit. + +### Observability + +The store publishes its own counters, so the bounds above can be tuned against +measurements rather than guesses: + +| Metric | What it tells you | +| --- | --- | +| `wire_uploads`, `wire_upload_bytes` | How much traffic actually takes the compressed fast path. | +| `wire_downloads` | Cache hits served byte-for-byte, with no re-encode. | +| `batch_zstd_passthroughs`, `batch_identity_decodes` | The same split for `BatchReadBlobs`. | +| `inline_commits`, `staged_commits` | How much of the upload mix stays in memory vs. hits `temp_path`. Tune `max_inline_commit_size` from this. | +| `staged_uploads_inflight`, `identity_ops_inflight` | Live occupancy against `max_concurrent_staged_uploads` / `max_concurrent_identity_ops`. Sustained saturation means requests are queuing. | +| `recompressions_applied`, `recompressions_rejected`, `recompressions_skipped_busy` | Whether recompression is earning its CPU, and whether its pool is too small. | +| `stage_timeouts`, `commit_timeouts` | Deadlines that fired. A nonzero `stage_timeouts` is either a trickling client or a `stage_timeout_s` set below real upload times. | + ## Choosing a terminal backend | Backend | Durability | Shared across instances | Typical role | @@ -161,6 +387,26 @@ instead of declaring the same backend twice: } ``` +## FAQ + + + + No — it never checks that an object in `fast` also made it to `slow`. + If an artifact must survive a fast-tier wipe, make sure the write + path covers both tiers deliberately. + + + Redis caps uploads at roughly 256-512 MB. Route large blobs to a + disk- or object-store backend with `size_partitioning` and keep Redis + serving the small, hot objects it's good at. + + + `existence_cache` and `size_partitioning` are CAS-only; + `completeness_checking` is AC-only. Misplacing one isn't a config + error — it surfaces as a confusing correctness bug. + + + ## What's next - [Configuration reference](/reference/nativelink-config) — every @@ -169,3 +415,6 @@ instead of declaring the same backend twice: fit alongside servers, schedulers, and workers. - [Configuration → Production](/configuration/production) — sharded, multi-region store topologies end to end. +- [`deployment-examples/docker-compose/local-storage-cas-zstd.json5`](https://github.com/TraceMachina/nativelink/blob/main/deployment-examples/docker-compose/local-storage-cas-zstd.json5) + — a complete zstd compression deployment, annotated with the namespace and + placement rules above. diff --git a/web/apps/docs/content/docs/reference/nativelink-config/v1.0.0.mdx b/web/apps/docs/content/docs/reference/nativelink-config/v1.0.0.mdx index cc4e46aca..f377ea362 100644 --- a/web/apps/docs/content/docs/reference/nativelink-config/v1.0.0.mdx +++ b/web/apps/docs/content/docs/reference/nativelink-config/v1.0.0.mdx @@ -127,8 +127,8 @@ It supports the following backends: } ``` -4. **`NetApp` ONTAP S3** - `NetApp` ONTAP S3 store will use ONTAP's S3-compatible storage as a backend +4. **NetApp ONTAP S3:** + NetApp ONTAP S3 store will use ONTAP's S3-compatible storage as a backend to store files. This store is specifically configured for ONTAP's S3 requirements including custom TLS configuration, credentials management, and proper vserver configuration. @@ -667,7 +667,7 @@ See [`experimental_cloud_object_store`](#experimental_cloud_object_store-1) for | Field | Type | Required | Default | Description | | --- | --- | --- | --- | --- | -| `backend` | [StoreSpec](#storespec) | Yes | — | The underlying store wrap around. All content will first flow through self before forwarding to backend. In the event there is an error detected in self, the connection to the backend will be terminated, and early termination should always cause updates to fail on the backend. | +| `backend` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | The underlying store wrap around. All content will first flow through self before forwarding to backend. In the event there is an error detected in self, the connection to the backend will be terminated, and early termination should always cause updates to fail on the backend. | | `verify_size` | boolean | — | `false` | If set the store will verify the size of the data before accepting an upload of data. | | `verify_hash` | boolean | — | `false` | If the data should be hashed and verify that the key matches the computed hash. The hash function is automatically determined based request and if not set will use the global default. | @@ -675,22 +675,22 @@ See [`experimental_cloud_object_store`](#experimental_cloud_object_store-1) for | Field | Type | Required | Default | Description | | --- | --- | --- | --- | --- | -| `backend` | [StoreSpec](#storespec) | Yes | — | The underlying store that will have it's results validated before sending to client. | -| `cas_store` | [StoreSpec](#storespec) | Yes | — | When a request is made, the results are decoded and all output digests/files are verified to exist in this CAS store before returning success. | +| `backend` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | The underlying store that will have it's results validated before sending to client. | +| `cas_store` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | When a request is made, the results are decoded and all output digests/files are verified to exist in this CAS store before returning success. | ## CompressionSpec | Field | Type | Required | Default | Description | | --- | --- | --- | --- | --- | -| `backend` | [StoreSpec](#storespec) | Yes | — | The underlying store wrap around. All content will first flow through self before forwarding to backend. In the event there is an error detected in self, the connection to the backend will be terminated, and early termination should always cause updates to fail on the backend. | +| `backend` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | The underlying store wrap around. All content will first flow through self before forwarding to backend. In the event there is an error detected in self, the connection to the backend will be terminated, and early termination should always cause updates to fail on the backend. | | `compression_algorithm` | [CompressionAlgorithm](#compressionalgorithm) | Yes | — | The compression algorithm to use. | ## DedupSpec | Field | Type | Required | Default | Description | | --- | --- | --- | --- | --- | -| `index_store` | [StoreSpec](#storespec) | Yes | — | Store used to store the index of each dedup slice. This store should generally be fast and small. | -| `content_store` | [StoreSpec](#storespec) | Yes | — | The store where the individual chunks will be uploaded. This store should generally be the slower & larger store. | +| `index_store` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | Store used to store the index of each dedup slice. This store should generally be fast and small. | +| `content_store` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | The store where the individual chunks will be uploaded. This store should generally be the slower & larger store. | | `min_size` | integer (uint32) | — | 65536 (64k) | Minimum size that a chunk will be when slicing up the content. Note: This setting can be increased to improve performance because it will actually not check this number of bytes when deciding where to partition the data. | | `normal_size` | integer (uint32) | — | 262144 (256k) | A best-effort attempt will be made to keep the average size of the chunks to this number. It is not a guarantee, but a slight attempt will be made. | | `max_size` | integer (uint32) | — | 524288 (512k) | Maximum size a chunk is allowed to be. | @@ -700,16 +700,16 @@ See [`experimental_cloud_object_store`](#experimental_cloud_object_store-1) for | Field | Type | Required | Default | Description | | --- | --- | --- | --- | --- | -| `backend` | [StoreSpec](#storespec) | Yes | — | The underlying store wrap around. All content will first flow through self before forwarding to backend. In the event there is an error detected in self, the connection to the backend will be terminated, and early termination should always cause updates to fail on the backend. | +| `backend` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | The underlying store wrap around. All content will first flow through self before forwarding to backend. In the event there is an error detected in self, the connection to the backend will be terminated, and early termination should always cause updates to fail on the backend. | | `eviction_policy` | [EvictionPolicy](#evictionpolicy) | — | — | Policy used to evict items out of the store. Failure to set this value will cause items to never be removed from the store causing infinite memory usage. | ## FastSlowSpec | Field | Type | Required | Default | Description | | --- | --- | --- | --- | --- | -| `fast` | [StoreSpec](#storespec) | Yes | — | Fast store that will be attempted to be contacted before reaching out to the `slow` store. | +| `fast` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | Fast store that will be attempted to be contacted before reaching out to the `slow` store. | | `fast_direction` | [StoreDirection](#storedirection) | — | `"both"` | How to handle the fast store. This can be useful to set to Get for worker nodes such that results are persisted to the slow store only. | -| `slow` | [StoreSpec](#storespec) | Yes | — | If the object does not exist in the `fast` store it will try to get it from this store. | +| `slow` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | If the object does not exist in the `fast` store it will try to get it from this store. | | `slow_direction` | [StoreDirection](#storedirection) | — | `"both"` | How to handle the slow store. This can be useful if creating a diode and you wish to have an upstream read only store. | ## ShardSpec @@ -740,8 +740,8 @@ See [`experimental_cloud_object_store`](#experimental_cloud_object_store-1) for | Field | Type | Required | Default | Description | | --- | --- | --- | --- | --- | | `size` | integer (uint64) | Yes | — | Size to partition the data on. | -| `lower_store` | [StoreSpec](#storespec) | Yes | — | Store to send data when object is < (less than) size. | -| `upper_store` | [StoreSpec](#storespec) | Yes | — | Store to send data when object is >= (less than eq) size. | +| `lower_store` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | Store to send data when object is < (less than) size. | +| `upper_store` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | Store to send data when object is >= (less than eq) size. | ## GrpcSpec @@ -970,553 +970,6 @@ The total delay is additive, so this example produces 9.525 to 15.875 s of total | `insecure_allow_http` | boolean | — | false | Allow unencrypted HTTP connections. Only use this for local testing. | | `disable_http2` | boolean | — | false | Disable HTTP/2 connections and only use HTTP/1.1. Default client configuration will have HTTP/1.1 and HTTP/2 enabled for connection schemes. HTTP/2 should be disabled if environments have poor support or performance related to HTTP/2. Safe to keep default unless underlying network environment, S3, or GCS API servers specify otherwise. | -## StoreSpec - -Plus exactly one of the following variants (the key selects the variant): - -### `memory` - -Memory store will store all data in a hash map in memory. - -**Example JSON5 config:** -```json5 -"memory": { - "eviction_policy": { - "max_bytes": "10mb", - } -} -``` - -**Type:** [MemorySpec](#memoryspec) - -### `experimental_cloud_object_store` - -A generic blob store that will store files on the cloud -provider. This configuration will never delete files, so you are -responsible for purging old files in other ways. -It supports the following backends: - -1. **Amazon S3:** - S3 store will use Amazon's S3 service as a backend to store - the files. This configuration can be used to share files - across multiple instances. Uses system certificates for TLS - verification via `rustls-platform-verifier`. - - **Example JSON5 config:** - ```json5 - "experimental_cloud_object_store": { - "provider": "aws", - "region": "eu-north-1", - "bucket": "crossplane-bucket-af79aeca9", - "key_prefix": "test-prefix-index/", - "retry": { - "max_retries": 6, - "delay": 0.3, - "jitter": 0.5 - }, - "multipart_max_concurrent_uploads": 10 - } - ``` - -2. **Google Cloud Storage:** - GCS store uses Google's GCS service as a backend to store - the files. This configuration can be used to share files - across multiple instances. - - **Example JSON5 config:** - ```json5 - "experimental_cloud_object_store": { - "provider": "gcs", - "bucket": "test-bucket", - "key_prefix": "test-prefix-index/", - "retry": { - "max_retries": 6, - "delay": 0.3, - "jitter": 0.5 - }, - "multipart_max_concurrent_uploads": 10 - } - ``` - -3. **Azure Blob Store:** - Azure Blob store will use Microsoft's Azure Blob service as a - backend to store the files. This configuration can be used to - share files across multiple instances. - - **Example JSON5 config:** - ```json5 - "experimental_cloud_object_store": { - "provider": "azure", - "account_name": "cloudshell1393657559", - "container": "simple-test-container", - "key_prefix": "folder/", - "retry": { - "max_retries": 6, - "delay": 0.3, - "jitter": 0.5 - }, - "multipart_max_concurrent_uploads": 10 - } - ``` - -4. **`NetApp` ONTAP S3** - `NetApp` ONTAP S3 store will use ONTAP's S3-compatible storage as a backend - to store files. This store is specifically configured for ONTAP's S3 requirements - including custom TLS configuration, credentials management, and proper vserver - configuration. - - This store uses AWS environment variables for credentials: - - `AWS_ACCESS_KEY_ID` - - `AWS_SECRET_ACCESS_KEY` - - `AWS_DEFAULT_REGION` - - **Example JSON5 config:** - ```json5 - "experimental_cloud_object_store": { - "provider": "ontap", - "endpoint": "https://ontap-s3-endpoint:443", - "vserver_name": "your-vserver", - "bucket": "your-bucket", - "root_certificates": "/path/to/certs.pem", // Optional - "key_prefix": "test-prefix/", // Optional - "retry": { - "max_retries": 6, - "delay": 0.3, - "jitter": 0.5 - }, - "multipart_max_concurrent_uploads": 10 - } - ``` - -**Type:** [ExperimentalCloudObjectSpec](#experimentalcloudobjectspec) - -### `ontap_s3_existence_cache` - -ONTAP S3 Existence Cache provides a caching layer on top of the ONTAP S3 store -to optimize repeated existence checks. It maintains an in-memory cache of object -digests and periodically syncs this cache to disk for persistence. - -The cache helps reduce latency for repeated calls to check object existence, -while still ensuring eventual consistency with the underlying ONTAP S3 store. - -Example JSON5 config: -```json5 -"ontap_s3_existence_cache": { - "index_path": "/path/to/cache/index.json", - "sync_interval_seconds": 300, - "backend": { - "endpoint": "https://ontap-s3-endpoint:443", - "vserver_name": "your-vserver", - "bucket": "your-bucket", - "key_prefix": "test-prefix/" - } -} -``` - -**Type:** [OntapS3ExistenceCacheSpec](#ontaps3existencecachespec) - -### `verify` - -Verify store is used to apply verifications to an underlying -store implementation. It is strongly encouraged to validate -as much data as you can before accepting data from a client, -failing to do so may cause the data in the store to be -populated with invalid data causing all kinds of problems. - -The suggested configuration is to have the CAS validate the -hash and size and the AC validate nothing. - -**Example JSON5 config:** -```json5 -"verify": { - "backend": { - "memory": { - "eviction_policy": { - "max_bytes": "500mb" - } - }, - }, - "verify_size": true, - "verify_hash": true -} -``` - -**Type:** [VerifySpec](#verifyspec) - -### `completeness_checking` - -Completeness checking store verifies if the -output files & folders exist in the CAS before forwarding -the request to the underlying store. -Note: This store should only be used on AC stores. - -**Example JSON5 config:** -```json5 -"completeness_checking": { - "backend": { - "filesystem": { - "content_path": "~/.cache/nativelink/content_path-ac", - "temp_path": "~/.cache/nativelink/tmp_path-ac", - "eviction_policy": { - "max_bytes": "500mb", - } - } - }, - "cas_store": { - "ref_store": { - "name": "CAS_MAIN_STORE" - } - } -} -``` - -**Type:** [CompletenessCheckingSpec](#completenesscheckingspec) - -### `compression` - -A compression store that will compress the data inbound and -outbound. There will be a non-trivial cost to compress and -decompress the data, but in many cases if the final store is -a store that requires network transport and/or storage space -is a concern it is often faster and more efficient to use this -store before those stores. - -**Example JSON5 config:** -```json5 -"compression": { - "compression_algorithm": { - "lz4": {} - }, - "backend": { - "filesystem": { - "content_path": "/tmp/nativelink/data/content_path-cas", - "temp_path": "/tmp/nativelink/data/tmp_path-cas", - "eviction_policy": { - "max_bytes": "2gb", - } - } - } -} -``` - -**Type:** [CompressionSpec](#compressionspec) - -### `dedup` - -A dedup store will take the inputs and run a rolling hash -algorithm on them to slice the input into smaller parts then -run a sha256 algorithm on the slice and if the object doesn't -already exist, upload the slice to the `content_store` using -a new digest of just the slice. Once all parts exist, an -Action-Cache-like digest will be built and uploaded to the -`index_store` which will contain a reference to each -chunk/digest of the uploaded file. Downloading a request will -first grab the index from the `index_store`, and forward the -download content of each chunk as if it were one file. - -This store is exceptionally good when the following conditions -are met: -* Content is mostly the same (inserts, updates, deletes are ok) -* Content is not compressed or encrypted -* Uploading or downloading from `content_store` is the bottleneck. - -Note: This store pairs well when used with `CompressionSpec` as -the `content_store`, but never put `DedupSpec` as the backend of -`CompressionSpec` as it will negate all the gains. - -Note: When running `.has()` on this store, it will only check -to see if the entry exists in the `index_store` and not check -if the individual chunks exist in the `content_store`. - -**Example JSON5 config:** -```json5 -"dedup": { - "index_store": { - "memory": { - "eviction_policy": { - "max_bytes": "1GB", - } - } - }, - "content_store": { - "compression": { - "compression_algorithm": { - "lz4": {} - }, - "backend": { - "fast_slow": { - "fast": { - "memory": { - "eviction_policy": { - "max_bytes": "500MB", - } - } - }, - "slow": { - "filesystem": { - "content_path": "/tmp/nativelink/data/content_path-content", - "temp_path": "/tmp/nativelink/data/tmp_path-content", - "eviction_policy": { - "max_bytes": "2gb" - } - } - } - } - } - } - } -} -``` - -**Type:** [DedupSpec](#dedupspec) - -### `existence_cache` - -Existence store will wrap around another store and cache calls -to has so that subsequent `has_with_results` calls will be -faster. This is useful for cases when you have a store that -is slow to respond to has calls. -Note: This store should only be used on CAS stores. - -**Example JSON5 config:** -```json5 -"existence_cache": { - "backend": { - "memory": { - "eviction_policy": { - "max_bytes": "500mb", - } - } - }, - // Note this is the existence store policy, not the backend policy - "eviction_policy": { - "max_seconds": 100, - } -} -``` - -**Type:** [ExistenceCacheSpec](#existencecachespec) - -### `fast_slow` - -`FastSlow` store will first try to fetch the data from the `fast` -store and then if it does not exist try the `slow` store. -When the object does exist in the `slow` store, it will copy -the data to the `fast` store while returning the data. -This store should be thought of as a store that "buffers" -the data to the `fast` store. -On uploads it will mirror data to both `fast` and `slow` stores. - -WARNING: If you need data to always exist in the `slow` store -for something like remote execution, be careful because this -store will never check to see if the objects exist in the -`slow` store if it exists in the `fast` store (i.e., it assumes -that if an object exists in the `fast` store it will exist in -the `slow` store). - -***Example JSON5 config:*** -```json5 -"fast_slow": { - "fast": { - "filesystem": { - "content_path": "/tmp/nativelink/data/content_path-index", - "temp_path": "/tmp/nativelink/data/tmp_path-index", - "eviction_policy": { - "max_bytes": "500mb", - } - } - }, - "slow": { - "filesystem": { - "content_path": "/tmp/nativelink/data/content_path-index", - "temp_path": "/tmp/nativelink/data/tmp_path-index", - "eviction_policy": { - "max_bytes": "500mb", - } - } - } -} -``` - -**Type:** [FastSlowSpec](#fastslowspec) - -### `shard` - -Shards the data to multiple stores. This is useful for cases -when you want to distribute the load across multiple stores. -The digest hash is used to determine which store to send the -data to. - -**Example JSON5 config:** -```json5 -"shard": { - "stores": [ - { - "store": { - "memory": { - "eviction_policy": { - "max_bytes": "10mb" - }, - }, - }, - "weight": 1 - }] -} -``` - -**Type:** [ShardSpec](#shardspec) - -### `filesystem` - -Stores the data on the filesystem. This store is designed for -local persistent storage. Restarts of this program should restore -the previous state, meaning anything uploaded will be persistent -as long as the filesystem integrity holds. - -**Example JSON5 config:** -```json5 -"filesystem": { - "content_path": "/tmp/nativelink/data-worker-test/content_path-cas", - "temp_path": "/tmp/nativelink/data-worker-test/tmp_path-cas", - "eviction_policy": { - "max_bytes": "10gb", - } -} -``` - -**Type:** [FilesystemSpec](#filesystemspec) - -### `ref_store` - -Store used to reference a store in the root store manager. -This is useful for cases when you want to share a store in different -nested stores. Example, you may want to share the same memory store -used for the action cache, but use a `FastSlowSpec` and have the fast -store also share the memory store for efficiency. - -**Example JSON5 config:** -```json5 -"ref_store": { - "name": "FS_CONTENT_STORE" -} -``` - -**Type:** [RefSpec](#refspec) - -### `size_partitioning` - -Uses the size field of the digest to separate which store to send the -data. This is useful for cases when you'd like to put small objects -in one store and large objects in another store. This should only be -used if the size field is the real size of the content, in other -words, don't use on AC (Action Cache) stores. Any store where you can -safely use `VerifySpec.verify_size = true`, this store should be safe -to use (i.e., CAS stores). - -**Example JSON5 config:** -```json5 -"size_partitioning": { - "size": "128mib", - "lower_store": { - "memory": { - "eviction_policy": { - "max_bytes": "${NATIVELINK_CAS_MEMORY_CONTENT_LIMIT:-100mb}" - } - } - }, - "upper_store": { - /// This store discards data larger than 128mib. - "noop": {} - } -} -``` - -**Type:** [SizePartitioningSpec](#sizepartitioningspec) - -### `grpc` - -This store will pass-through calls to another GRPC store. This store -is not designed to be used as a sub-store of another store, but it -does satisfy the interface and will likely work. - -One major GOTCHA is that some stores use a special function on this -store to get the size of the underlying object, which is only reliable -when this store is serving the a CAS store, not an AC store. If using -this store directly without being a child of any store there are no -side effects and is the most efficient way to use it. - -**Example JSON5 config:** -```json5 -"grpc": { - "instance_name": "main", - "endpoints": [ - {"address": "grpc://${CAS_ENDPOINT:-127.0.0.1}:50051"} - ], - "connections_per_endpoint": "5", - "rpc_timeout_s": "5m", - "store_type": "ac" -} -``` - -**Type:** [GrpcSpec](#grpcspec) - -### `redis_store` - -Stores data in any stores compatible with Redis APIs. - -Pairs well with `SizePartitioning` and/or `FastSlow` stores. -Ideal for accepting small object sizes as most Redis store -services have a max file upload of between 256Mb-512Mb. - -**Example JSON5 config:** -```json5 -"redis_store": { - "addresses": [ - "redis://127.0.0.1:6379/", - ], - "max_client_permits": 1000, -} -``` - -**Type:** [RedisSpec](#redisspec) - -### `noop` - -Noop store is a store that sends streams into the void and all data -retrieval will return 404 (`NotFound`). This can be useful for cases -where you may need to partition your data and part of your data needs -to be discarded. - -**Example JSON5 config:** -```json5 -"noop": {} -``` - -**Type:** [NoopSpec](#noopspec) - -### `experimental_mongo` - -Experimental `MongoDB` store implementation. - -This store uses `MongoDB` as a backend for storing data. It supports -both CAS (Content Addressable Storage) and scheduler data with -optional change streams for real-time updates. - -**Example JSON5 config:** -```json5 -"experimental_mongo": { - "connection_string": "mongodb://localhost:27017", - "database": "nativelink", - "cas_collection": "cas", - "key_prefix": "cas:", - "read_chunk_size": 65536, - "max_concurrent_uploads": 10, - "enable_change_streams": false -} -``` - -**Type:** [ExperimentalMongoSpec](#experimentalmongospec) - ## CompressionAlgorithm Plus exactly one of the following variants (the key selects the variant): @@ -1548,7 +1001,7 @@ Configuration for an individual shard of the store. | Field | Type | Required | Default | Description | | --- | --- | --- | --- | --- | -| `store` | [StoreSpec](#storespec) | Yes | — | Store to shard the data to. | +| `store` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | Store to shard the data to. | | `weight` | integer (uint32) | — | 1 | The weight of the store. This is used to determine how much data should be sent to the store. The actual percentage is the sum of all the store's weights divided by the individual store's weight. | ## GrpcEndpoint diff --git a/web/apps/docs/content/docs/reference/nativelink-config/v1.1.0.mdx b/web/apps/docs/content/docs/reference/nativelink-config/v1.1.0.mdx index 410d1a6b1..a1abfdf37 100644 --- a/web/apps/docs/content/docs/reference/nativelink-config/v1.1.0.mdx +++ b/web/apps/docs/content/docs/reference/nativelink-config/v1.1.0.mdx @@ -127,8 +127,8 @@ It supports the following backends: } ``` -4. **`NetApp` ONTAP S3** - `NetApp` ONTAP S3 store will use ONTAP's S3-compatible storage as a backend +4. **NetApp ONTAP S3:** + NetApp ONTAP S3 store will use ONTAP's S3-compatible storage as a backend to store files. This store is specifically configured for ONTAP's S3 requirements including custom TLS configuration, credentials management, and proper vserver configuration. @@ -677,7 +677,7 @@ See [`experimental_cloud_object_store`](#experimental_cloud_object_store-1) for | Field | Type | Required | Default | Description | | --- | --- | --- | --- | --- | -| `backend` | [StoreSpec](#storespec) | Yes | — | The underlying store wrap around. All content will first flow through self before forwarding to backend. In the event there is an error detected in self, the connection to the backend will be terminated, and early termination should always cause updates to fail on the backend. | +| `backend` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | The underlying store wrap around. All content will first flow through self before forwarding to backend. In the event there is an error detected in self, the connection to the backend will be terminated, and early termination should always cause updates to fail on the backend. | | `verify_size` | boolean | — | `false` | If set the store will verify the size of the data before accepting an upload of data. | | `verify_hash` | boolean | — | `false` | If the data should be hashed and verify that the key matches the computed hash. The hash function is automatically determined based request and if not set will use the global default. | @@ -685,22 +685,22 @@ See [`experimental_cloud_object_store`](#experimental_cloud_object_store-1) for | Field | Type | Required | Default | Description | | --- | --- | --- | --- | --- | -| `backend` | [StoreSpec](#storespec) | Yes | — | The underlying store that will have it's results validated before sending to client. | -| `cas_store` | [StoreSpec](#storespec) | Yes | — | When a request is made, the results are decoded and all output digests/files are verified to exist in this CAS store before returning success. | +| `backend` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | The underlying store that will have it's results validated before sending to client. | +| `cas_store` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | When a request is made, the results are decoded and all output digests/files are verified to exist in this CAS store before returning success. | ## CompressionSpec | Field | Type | Required | Default | Description | | --- | --- | --- | --- | --- | -| `backend` | [StoreSpec](#storespec) | Yes | — | The underlying store wrap around. All content will first flow through self before forwarding to backend. In the event there is an error detected in self, the connection to the backend will be terminated, and early termination should always cause updates to fail on the backend. | +| `backend` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | The underlying store wrap around. All content will first flow through self before forwarding to backend. In the event there is an error detected in self, the connection to the backend will be terminated, and early termination should always cause updates to fail on the backend. | | `compression_algorithm` | [CompressionAlgorithm](#compressionalgorithm) | Yes | — | The compression algorithm to use. | ## DedupSpec | Field | Type | Required | Default | Description | | --- | --- | --- | --- | --- | -| `index_store` | [StoreSpec](#storespec) | Yes | — | Store used to store the index of each dedup slice. This store should generally be fast and small. | -| `content_store` | [StoreSpec](#storespec) | Yes | — | The store where the individual chunks will be uploaded. This store should generally be the slower & larger store. | +| `index_store` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | Store used to store the index of each dedup slice. This store should generally be fast and small. | +| `content_store` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | The store where the individual chunks will be uploaded. This store should generally be the slower & larger store. | | `min_size` | integer (uint32) | — | 65536 (64k) | Minimum size that a chunk will be when slicing up the content. Note: This setting can be increased to improve performance because it will actually not check this number of bytes when deciding where to partition the data. | | `normal_size` | integer (uint32) | — | 262144 (256k) | A best-effort attempt will be made to keep the average size of the chunks to this number. It is not a guarantee, but a slight attempt will be made. | | `max_size` | integer (uint32) | — | 524288 (512k) | Maximum size a chunk is allowed to be. | @@ -710,16 +710,16 @@ See [`experimental_cloud_object_store`](#experimental_cloud_object_store-1) for | Field | Type | Required | Default | Description | | --- | --- | --- | --- | --- | -| `backend` | [StoreSpec](#storespec) | Yes | — | The underlying store wrap around. All content will first flow through self before forwarding to backend. In the event there is an error detected in self, the connection to the backend will be terminated, and early termination should always cause updates to fail on the backend. | +| `backend` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | The underlying store wrap around. All content will first flow through self before forwarding to backend. In the event there is an error detected in self, the connection to the backend will be terminated, and early termination should always cause updates to fail on the backend. | | `eviction_policy` | [EvictionPolicy](#evictionpolicy) | — | — | Policy used to evict items out of the store. Failure to set this value will cause items to never be removed from the store causing infinite memory usage. | ## FastSlowSpec | Field | Type | Required | Default | Description | | --- | --- | --- | --- | --- | -| `fast` | [StoreSpec](#storespec) | Yes | — | Fast store that will be attempted to be contacted before reaching out to the `slow` store. | +| `fast` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | Fast store that will be attempted to be contacted before reaching out to the `slow` store. | | `fast_direction` | [StoreDirection](#storedirection) | — | `"both"` | How to handle the fast store. This can be useful to set to Get for worker nodes such that results are persisted to the slow store only. | -| `slow` | [StoreSpec](#storespec) | Yes | — | If the object does not exist in the `fast` store it will try to get it from this store. | +| `slow` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | If the object does not exist in the `fast` store it will try to get it from this store. | | `slow_direction` | [StoreDirection](#storedirection) | — | `"both"` | How to handle the slow store. This can be useful if creating a diode and you wish to have an upstream read only store. | ## ShardSpec @@ -750,8 +750,8 @@ See [`experimental_cloud_object_store`](#experimental_cloud_object_store-1) for | Field | Type | Required | Default | Description | | --- | --- | --- | --- | --- | | `size` | integer (uint64) | Yes | — | Size to partition the data on. | -| `lower_store` | [StoreSpec](#storespec) | Yes | — | Store to send data when object is < (less than) size. | -| `upper_store` | [StoreSpec](#storespec) | Yes | — | Store to send data when object is >= (less than eq) size. | +| `lower_store` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | Store to send data when object is < (less than) size. | +| `upper_store` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | Store to send data when object is >= (less than eq) size. | ## GrpcSpec @@ -986,563 +986,6 @@ The total delay is additive, so this example produces 9.525 to 15.875 s of total | `insecure_allow_http` | boolean | — | false | Allow unencrypted HTTP connections. Only use this for local testing. | | `disable_http2` | boolean | — | false | Disable HTTP/2 connections and only use HTTP/1.1. Default client configuration will have HTTP/1.1 and HTTP/2 enabled for connection schemes. HTTP/2 should be disabled if environments have poor support or performance related to HTTP/2. Safe to keep default unless underlying network environment, S3, or GCS API servers specify otherwise. | -## StoreSpec - -Plus exactly one of the following variants (the key selects the variant): - -### `memory` - -Memory store will store all data in a hash map in memory. - -**Example JSON5 config:** -```json5 -"memory": { - "eviction_policy": { - "max_bytes": "10mb", - } -} -``` - -**Type:** [MemorySpec](#memoryspec) - -### `experimental_cloud_object_store` - -A generic blob store that will store files on the cloud -provider. This configuration will never delete files, so you are -responsible for purging old files in other ways. -It supports the following backends: - -1. **Amazon S3:** - S3 store will use Amazon's S3 service as a backend to store - the files. This configuration can be used to share files - across multiple instances. Uses system certificates for TLS - verification via `rustls-platform-verifier`. - - **Example JSON5 config:** - ```json5 - "experimental_cloud_object_store": { - "provider": "aws", - "region": "eu-north-1", - "bucket": "crossplane-bucket-af79aeca9", - "key_prefix": "test-prefix-index/", - "retry": { - "max_retries": 6, - "delay": 0.3, - "jitter": 0.5 - }, - "multipart_max_concurrent_uploads": 10 - } - ``` - -2. **Google Cloud Storage:** - GCS store uses Google's GCS service as a backend to store - the files. This configuration can be used to share files - across multiple instances. - - **Example JSON5 config:** - ```json5 - "experimental_cloud_object_store": { - "provider": "gcs", - "bucket": "test-bucket", - "key_prefix": "test-prefix-index/", - "retry": { - "max_retries": 6, - "delay": 0.3, - "jitter": 0.5 - }, - "multipart_max_concurrent_uploads": 10 - } - ``` - -3. **Azure Blob Store:** - Azure Blob store will use Microsoft's Azure Blob service as a - backend to store the files. This configuration can be used to - share files across multiple instances. - - **Example JSON5 config:** - ```json5 - "experimental_cloud_object_store": { - "provider": "azure", - "account_name": "cloudshell1393657559", - "container": "simple-test-container", - "key_prefix": "folder/", - "retry": { - "max_retries": 6, - "delay": 0.3, - "jitter": 0.5 - }, - "multipart_max_concurrent_uploads": 10 - } - ``` - -4. **`NetApp` ONTAP S3** - `NetApp` ONTAP S3 store will use ONTAP's S3-compatible storage as a backend - to store files. This store is specifically configured for ONTAP's S3 requirements - including custom TLS configuration, credentials management, and proper vserver - configuration. - - This store uses AWS environment variables for credentials: - - `AWS_ACCESS_KEY_ID` - - `AWS_SECRET_ACCESS_KEY` - - `AWS_DEFAULT_REGION` - - **Example JSON5 config:** - ```json5 - "experimental_cloud_object_store": { - "provider": "ontap", - "endpoint": "https://ontap-s3-endpoint:443", - "vserver_name": "your-vserver", - "bucket": "your-bucket", - "root_certificates": "/path/to/certs.pem", // Optional - "key_prefix": "test-prefix/", // Optional - "retry": { - "max_retries": 6, - "delay": 0.3, - "jitter": 0.5 - }, - "multipart_max_concurrent_uploads": 10 - } - ``` - -**Type:** [ExperimentalCloudObjectSpec](#experimentalcloudobjectspec) - -### `ontap_s3_existence_cache` - -ONTAP S3 Existence Cache provides a caching layer on top of the ONTAP S3 store -to optimize repeated existence checks. It maintains an in-memory cache of object -digests and periodically syncs this cache to disk for persistence. - -The cache helps reduce latency for repeated calls to check object existence, -while still ensuring eventual consistency with the underlying ONTAP S3 store. - -Example JSON5 config: -```json5 -"ontap_s3_existence_cache": { - "index_path": "/path/to/cache/index.json", - "sync_interval_seconds": 300, - "backend": { - "endpoint": "https://ontap-s3-endpoint:443", - "vserver_name": "your-vserver", - "bucket": "your-bucket", - "key_prefix": "test-prefix/" - } -} -``` - -**Type:** [OntapS3ExistenceCacheSpec](#ontaps3existencecachespec) - -### `verify` - -Verify store is used to apply verifications to an underlying -store implementation. It is strongly encouraged to validate -as much data as you can before accepting data from a client, -failing to do so may cause the data in the store to be -populated with invalid data causing all kinds of problems. - -The suggested configuration is to have the CAS validate the -hash and size and the AC validate nothing. - -**Example JSON5 config:** -```json5 -"verify": { - "backend": { - "memory": { - "eviction_policy": { - "max_bytes": "500mb" - } - }, - }, - "verify_size": true, - "verify_hash": true -} -``` - -**Type:** [VerifySpec](#verifyspec) - -### `completeness_checking` - -Completeness checking store verifies if the -output files & folders exist in the CAS before forwarding -the request to the underlying store. -Note: This store should only be used on AC stores. - -**Example JSON5 config:** -```json5 -"completeness_checking": { - "backend": { - "filesystem": { - "content_path": "~/.cache/nativelink/content_path-ac", - "temp_path": "~/.cache/nativelink/tmp_path-ac", - "eviction_policy": { - "max_bytes": "500mb", - } - } - }, - "cas_store": { - "ref_store": { - "name": "CAS_MAIN_STORE" - } - } -} -``` - -**Type:** [CompletenessCheckingSpec](#completenesscheckingspec) - -### `compression` - -A compression store that will compress the data inbound and -outbound. There will be a non-trivial cost to compress and -decompress the data, but in many cases if the final store is -a store that requires network transport and/or storage space -is a concern it is often faster and more efficient to use this -store before those stores. - -**Example JSON5 config:** -```json5 -"compression": { - "compression_algorithm": { - "lz4": {} - }, - "backend": { - "filesystem": { - "content_path": "/tmp/nativelink/data/content_path-cas", - "temp_path": "/tmp/nativelink/data/tmp_path-cas", - "eviction_policy": { - "max_bytes": "2gb", - } - } - } -} -``` - -**Type:** [CompressionSpec](#compressionspec) - -### `dedup` - -A dedup store will take the inputs and run a rolling hash -algorithm on them to slice the input into smaller parts then -run a sha256 algorithm on the slice and if the object doesn't -already exist, upload the slice to the `content_store` using -a new digest of just the slice. Once all parts exist, an -Action-Cache-like digest will be built and uploaded to the -`index_store` which will contain a reference to each -chunk/digest of the uploaded file. Downloading a request will -first grab the index from the `index_store`, and forward the -download content of each chunk as if it were one file. - -This store is exceptionally good when the following conditions -are met: -* Content is mostly the same (inserts, updates, deletes are ok) -* Content is not compressed or encrypted -* Uploading or downloading from `content_store` is the bottleneck. - -Note: This store pairs well when used with `CompressionSpec` as -the `content_store`, but never put `DedupSpec` as the backend of -`CompressionSpec` as it will negate all the gains. - -Note: When running `.has()` on this store, it will only check -to see if the entry exists in the `index_store` and not check -if the individual chunks exist in the `content_store`. - -**Example JSON5 config:** -```json5 -"dedup": { - "index_store": { - "memory": { - "eviction_policy": { - "max_bytes": "1GB", - } - } - }, - "content_store": { - "compression": { - "compression_algorithm": { - "lz4": {} - }, - "backend": { - "fast_slow": { - "fast": { - "memory": { - "eviction_policy": { - "max_bytes": "500MB", - } - } - }, - "slow": { - "filesystem": { - "content_path": "/tmp/nativelink/data/content_path-content", - "temp_path": "/tmp/nativelink/data/tmp_path-content", - "eviction_policy": { - "max_bytes": "2gb" - } - } - } - } - } - } - } -} -``` - -**Type:** [DedupSpec](#dedupspec) - -### `existence_cache` - -Existence store will wrap around another store and cache calls -to has so that subsequent `has_with_results` calls will be -faster. This is useful for cases when you have a store that -is slow to respond to has calls. -Note: This store should only be used on CAS stores. - -**Example JSON5 config:** -```json5 -"existence_cache": { - "backend": { - "memory": { - "eviction_policy": { - "max_bytes": "500mb", - } - } - }, - // Note this is the existence store policy, not the backend policy - "eviction_policy": { - "max_seconds": 100, - } -} -``` - -**Type:** [ExistenceCacheSpec](#existencecachespec) - -### `fast_slow` - -`FastSlow` store will first try to fetch the data from the `fast` -store and then if it does not exist try the `slow` store. -When the object does exist in the `slow` store, it will copy -the data to the `fast` store while returning the data. -This store should be thought of as a store that "buffers" -the data to the `fast` store. -On uploads it will mirror data to both `fast` and `slow` stores. - -WARNING: If you need data to always exist in the `slow` store -for something like remote execution, be careful because this -store will never check to see if the objects exist in the -`slow` store if it exists in the `fast` store (i.e., it assumes -that if an object exists in the `fast` store it will exist in -the `slow` store). - -***Example JSON5 config:*** -```json5 -"fast_slow": { - "fast": { - "filesystem": { - "content_path": "/tmp/nativelink/data/content_path-index", - "temp_path": "/tmp/nativelink/data/tmp_path-index", - "eviction_policy": { - "max_bytes": "500mb", - } - } - }, - "slow": { - "filesystem": { - "content_path": "/tmp/nativelink/data/content_path-index", - "temp_path": "/tmp/nativelink/data/tmp_path-index", - "eviction_policy": { - "max_bytes": "500mb", - } - } - } -} -``` - -**Type:** [FastSlowSpec](#fastslowspec) - -### `shard` - -Shards the data to multiple stores. This is useful for cases -when you want to distribute the load across multiple stores. -The digest hash is used to determine which store to send the -data to. - -**Example JSON5 config:** -```json5 -"shard": { - "stores": [ - { - "store": { - "memory": { - "eviction_policy": { - "max_bytes": "10mb" - }, - }, - }, - "weight": 1 - }] -} -``` - -**Type:** [ShardSpec](#shardspec) - -### `filesystem` - -Stores the data on the filesystem. This store is designed for -local persistent storage. Restarts of this program should restore -the previous state, meaning anything uploaded will be persistent -as long as the filesystem integrity holds. - -**Example JSON5 config:** -```json5 -"filesystem": { - "content_path": "/tmp/nativelink/data-worker-test/content_path-cas", - "temp_path": "/tmp/nativelink/data-worker-test/tmp_path-cas", - "eviction_policy": { - "max_bytes": "10gb", - } -} -``` - -**Type:** [FilesystemSpec](#filesystemspec) - -### `ref_store` - -Store used to reference a store in the root store manager. -This is useful for cases when you want to share a store in different -nested stores. Example, you may want to share the same memory store -used for the action cache, but use a `FastSlowSpec` and have the fast -store also share the memory store for efficiency. - -**Example JSON5 config:** -```json5 -"ref_store": { - "name": "FS_CONTENT_STORE" -} -``` - -**Type:** [RefSpec](#refspec) - -### `size_partitioning` - -Uses the size field of the digest to separate which store to send the -data. This is useful for cases when you'd like to put small objects -in one store and large objects in another store. This should only be -used if the size field is the real size of the content, in other -words, don't use on AC (Action Cache) stores. Any store where you can -safely use `VerifySpec.verify_size = true`, this store should be safe -to use (i.e., CAS stores). - -**Example JSON5 config:** -```json5 -"size_partitioning": { - "size": "128mib", - "lower_store": { - "memory": { - "eviction_policy": { - "max_bytes": "${NATIVELINK_CAS_MEMORY_CONTENT_LIMIT:-100mb}" - } - } - }, - "upper_store": { - /// This store discards data larger than 128mib. - "noop": {} - } -} -``` - -**Type:** [SizePartitioningSpec](#sizepartitioningspec) - -### `grpc` - -This store will pass-through calls to another GRPC store. This store -is not designed to be used as a sub-store of another store, but it -does satisfy the interface and will likely work. - -One major GOTCHA is that some stores use a special function on this -store to get the size of the underlying object, which is only reliable -when this store is serving the a CAS store, not an AC store. If using -this store directly without being a child of any store there are no -side effects and is the most efficient way to use it. - -**Example JSON5 config:** -```json5 -"grpc": { - "instance_name": "main", - "endpoints": [ - {"address": "grpc://${CAS_ENDPOINT:-127.0.0.1}:50051"} - ], - "connections_per_endpoint": "5", - "rpc_timeout_s": "5m", - "store_type": "ac", - // Static headers attached to every outgoing request to the upstream - // remote cache. Useful for fixed service-account credentials. - "headers": { - "authorization": "Bearer my-static-token" - }, - // Header names to copy from the inbound client request and forward to - // the upstream remote cache. Use this to pass through dynamic - // credentials such as a JWT sent by the build client. - "forward_headers": ["authorization", "x-custom-token"] -} -``` - -**Type:** [GrpcSpec](#grpcspec) - -### `redis_store` - -Stores data in any stores compatible with Redis APIs. - -Pairs well with `SizePartitioning` and/or `FastSlow` stores. -Ideal for accepting small object sizes as most Redis store -services have a max file upload of between 256Mb-512Mb. - -**Example JSON5 config:** -```json5 -"redis_store": { - "addresses": [ - "redis://127.0.0.1:6379/", - ], - "max_client_permits": 1000, -} -``` - -**Type:** [RedisSpec](#redisspec) - -### `noop` - -Noop store is a store that sends streams into the void and all data -retrieval will return 404 (`NotFound`). This can be useful for cases -where you may need to partition your data and part of your data needs -to be discarded. - -**Example JSON5 config:** -```json5 -"noop": {} -``` - -**Type:** [NoopSpec](#noopspec) - -### `experimental_mongo` - -Experimental `MongoDB` store implementation. - -This store uses `MongoDB` as a backend for storing data. It supports -both CAS (Content Addressable Storage) and scheduler data with -optional change streams for real-time updates. - -**Example JSON5 config:** -```json5 -"experimental_mongo": { - "connection_string": "mongodb://localhost:27017", - "database": "nativelink", - "cas_collection": "cas", - "key_prefix": "cas:", - "read_chunk_size": 65536, - "max_concurrent_uploads": 10, - "enable_change_streams": false, - "max_requests": "100" -} -``` - -**Type:** [ExperimentalMongoSpec](#experimentalmongospec) - ## CompressionAlgorithm Plus exactly one of the following variants (the key selects the variant): @@ -1574,7 +1017,7 @@ Configuration for an individual shard of the store. | Field | Type | Required | Default | Description | | --- | --- | --- | --- | --- | -| `store` | [StoreSpec](#storespec) | Yes | — | Store to shard the data to. | +| `store` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | Store to shard the data to. | | `weight` | integer (uint32) | — | 1 | The weight of the store. This is used to determine how much data should be sent to the store. The actual percentage is the sum of all the store's weights divided by the individual store's weight. | ## GrpcEndpoint diff --git a/web/apps/docs/content/docs/reference/nativelink-config/v1.2.0.mdx b/web/apps/docs/content/docs/reference/nativelink-config/v1.2.0.mdx index b25c586b0..878fb9b56 100644 --- a/web/apps/docs/content/docs/reference/nativelink-config/v1.2.0.mdx +++ b/web/apps/docs/content/docs/reference/nativelink-config/v1.2.0.mdx @@ -127,8 +127,8 @@ It supports the following backends: } ``` -4. **`NetApp` ONTAP S3** - `NetApp` ONTAP S3 store will use ONTAP's S3-compatible storage as a backend +4. **NetApp ONTAP S3:** + NetApp ONTAP S3 store will use ONTAP's S3-compatible storage as a backend to store files. This store is specifically configured for ONTAP's S3 requirements including custom TLS configuration, credentials management, and proper vserver configuration. @@ -677,7 +677,7 @@ See [`experimental_cloud_object_store`](#experimental_cloud_object_store-1) for | Field | Type | Required | Default | Description | | --- | --- | --- | --- | --- | -| `backend` | [StoreSpec](#storespec) | Yes | — | The underlying store wrap around. All content will first flow through self before forwarding to backend. In the event there is an error detected in self, the connection to the backend will be terminated, and early termination should always cause updates to fail on the backend. | +| `backend` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | The underlying store wrap around. All content will first flow through self before forwarding to backend. In the event there is an error detected in self, the connection to the backend will be terminated, and early termination should always cause updates to fail on the backend. | | `verify_size` | boolean | — | `false` | If set the store will verify the size of the data before accepting an upload of data. | | `verify_hash` | boolean | — | `false` | If the data should be hashed and verify that the key matches the computed hash. The hash function is automatically determined based request and if not set will use the global default. | @@ -685,22 +685,22 @@ See [`experimental_cloud_object_store`](#experimental_cloud_object_store-1) for | Field | Type | Required | Default | Description | | --- | --- | --- | --- | --- | -| `backend` | [StoreSpec](#storespec) | Yes | — | The underlying store that will have it's results validated before sending to client. | -| `cas_store` | [StoreSpec](#storespec) | Yes | — | When a request is made, the results are decoded and all output digests/files are verified to exist in this CAS store before returning success. | +| `backend` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | The underlying store that will have it's results validated before sending to client. | +| `cas_store` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | When a request is made, the results are decoded and all output digests/files are verified to exist in this CAS store before returning success. | ## CompressionSpec | Field | Type | Required | Default | Description | | --- | --- | --- | --- | --- | -| `backend` | [StoreSpec](#storespec) | Yes | — | The underlying store wrap around. All content will first flow through self before forwarding to backend. In the event there is an error detected in self, the connection to the backend will be terminated, and early termination should always cause updates to fail on the backend. | +| `backend` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | The underlying store wrap around. All content will first flow through self before forwarding to backend. In the event there is an error detected in self, the connection to the backend will be terminated, and early termination should always cause updates to fail on the backend. | | `compression_algorithm` | [CompressionAlgorithm](#compressionalgorithm) | Yes | — | The compression algorithm to use. | ## DedupSpec | Field | Type | Required | Default | Description | | --- | --- | --- | --- | --- | -| `index_store` | [StoreSpec](#storespec) | Yes | — | Store used to store the index of each dedup slice. This store should generally be fast and small. | -| `content_store` | [StoreSpec](#storespec) | Yes | — | The store where the individual chunks will be uploaded. This store should generally be the slower & larger store. | +| `index_store` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | Store used to store the index of each dedup slice. This store should generally be fast and small. | +| `content_store` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | The store where the individual chunks will be uploaded. This store should generally be the slower & larger store. | | `min_size` | integer (uint32) | — | 65536 (64k) | Minimum size that a chunk will be when slicing up the content. Note: This setting can be increased to improve performance because it will actually not check this number of bytes when deciding where to partition the data. | | `normal_size` | integer (uint32) | — | 262144 (256k) | A best-effort attempt will be made to keep the average size of the chunks to this number. It is not a guarantee, but a slight attempt will be made. | | `max_size` | integer (uint32) | — | 524288 (512k) | Maximum size a chunk is allowed to be. | @@ -710,16 +710,16 @@ See [`experimental_cloud_object_store`](#experimental_cloud_object_store-1) for | Field | Type | Required | Default | Description | | --- | --- | --- | --- | --- | -| `backend` | [StoreSpec](#storespec) | Yes | — | The underlying store wrap around. All content will first flow through self before forwarding to backend. In the event there is an error detected in self, the connection to the backend will be terminated, and early termination should always cause updates to fail on the backend. | +| `backend` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | The underlying store wrap around. All content will first flow through self before forwarding to backend. In the event there is an error detected in self, the connection to the backend will be terminated, and early termination should always cause updates to fail on the backend. | | `eviction_policy` | [EvictionPolicy](#evictionpolicy) | — | — | Policy used to evict items out of the store. Failure to set this value will cause items to never be removed from the store causing infinite memory usage. | ## FastSlowSpec | Field | Type | Required | Default | Description | | --- | --- | --- | --- | --- | -| `fast` | [StoreSpec](#storespec) | Yes | — | Fast store that will be attempted to be contacted before reaching out to the `slow` store. | +| `fast` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | Fast store that will be attempted to be contacted before reaching out to the `slow` store. | | `fast_direction` | [StoreDirection](#storedirection) | — | `"both"` | How to handle the fast store. This can be useful to set to Get for worker nodes such that results are persisted to the slow store only. | -| `slow` | [StoreSpec](#storespec) | Yes | — | If the object does not exist in the `fast` store it will try to get it from this store. | +| `slow` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | If the object does not exist in the `fast` store it will try to get it from this store. | | `slow_direction` | [StoreDirection](#storedirection) | — | `"both"` | How to handle the slow store. This can be useful if creating a diode and you wish to have an upstream read only store. | ## ShardSpec @@ -750,8 +750,8 @@ See [`experimental_cloud_object_store`](#experimental_cloud_object_store-1) for | Field | Type | Required | Default | Description | | --- | --- | --- | --- | --- | | `size` | integer (uint64) | Yes | — | Size to partition the data on. | -| `lower_store` | [StoreSpec](#storespec) | Yes | — | Store to send data when object is < (less than) size. | -| `upper_store` | [StoreSpec](#storespec) | Yes | — | Store to send data when object is >= (less than eq) size. | +| `lower_store` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | Store to send data when object is < (less than) size. | +| `upper_store` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | Store to send data when object is >= (less than eq) size. | ## GrpcSpec @@ -986,563 +986,6 @@ The total delay is additive, so this example produces 9.525 to 15.875 s of total | `insecure_allow_http` | boolean | — | false | Allow unencrypted HTTP connections. Only use this for local testing. | | `disable_http2` | boolean | — | false | Disable HTTP/2 connections and only use HTTP/1.1. Default client configuration will have HTTP/1.1 and HTTP/2 enabled for connection schemes. HTTP/2 should be disabled if environments have poor support or performance related to HTTP/2. Safe to keep default unless underlying network environment, S3, or GCS API servers specify otherwise. | -## StoreSpec - -Plus exactly one of the following variants (the key selects the variant): - -### `memory` - -Memory store will store all data in a hash map in memory. - -**Example JSON5 config:** -```json5 -"memory": { - "eviction_policy": { - "max_bytes": "10mb", - } -} -``` - -**Type:** [MemorySpec](#memoryspec) - -### `experimental_cloud_object_store` - -A generic blob store that will store files on the cloud -provider. This configuration will never delete files, so you are -responsible for purging old files in other ways. -It supports the following backends: - -1. **Amazon S3:** - S3 store will use Amazon's S3 service as a backend to store - the files. This configuration can be used to share files - across multiple instances. Uses system certificates for TLS - verification via `rustls-platform-verifier`. - - **Example JSON5 config:** - ```json5 - "experimental_cloud_object_store": { - "provider": "aws", - "region": "eu-north-1", - "bucket": "crossplane-bucket-af79aeca9", - "key_prefix": "test-prefix-index/", - "retry": { - "max_retries": 6, - "delay": 0.3, - "jitter": 0.5 - }, - "multipart_max_concurrent_uploads": 10 - } - ``` - -2. **Google Cloud Storage:** - GCS store uses Google's GCS service as a backend to store - the files. This configuration can be used to share files - across multiple instances. - - **Example JSON5 config:** - ```json5 - "experimental_cloud_object_store": { - "provider": "gcs", - "bucket": "test-bucket", - "key_prefix": "test-prefix-index/", - "retry": { - "max_retries": 6, - "delay": 0.3, - "jitter": 0.5 - }, - "multipart_max_concurrent_uploads": 10 - } - ``` - -3. **Azure Blob Store:** - Azure Blob store will use Microsoft's Azure Blob service as a - backend to store the files. This configuration can be used to - share files across multiple instances. - - **Example JSON5 config:** - ```json5 - "experimental_cloud_object_store": { - "provider": "azure", - "account_name": "cloudshell1393657559", - "container": "simple-test-container", - "key_prefix": "folder/", - "retry": { - "max_retries": 6, - "delay": 0.3, - "jitter": 0.5 - }, - "multipart_max_concurrent_uploads": 10 - } - ``` - -4. **`NetApp` ONTAP S3** - `NetApp` ONTAP S3 store will use ONTAP's S3-compatible storage as a backend - to store files. This store is specifically configured for ONTAP's S3 requirements - including custom TLS configuration, credentials management, and proper vserver - configuration. - - This store uses AWS environment variables for credentials: - - `AWS_ACCESS_KEY_ID` - - `AWS_SECRET_ACCESS_KEY` - - `AWS_DEFAULT_REGION` - - **Example JSON5 config:** - ```json5 - "experimental_cloud_object_store": { - "provider": "ontap", - "endpoint": "https://ontap-s3-endpoint:443", - "vserver_name": "your-vserver", - "bucket": "your-bucket", - "root_certificates": "/path/to/certs.pem", // Optional - "key_prefix": "test-prefix/", // Optional - "retry": { - "max_retries": 6, - "delay": 0.3, - "jitter": 0.5 - }, - "multipart_max_concurrent_uploads": 10 - } - ``` - -**Type:** [ExperimentalCloudObjectSpec](#experimentalcloudobjectspec) - -### `ontap_s3_existence_cache` - -ONTAP S3 Existence Cache provides a caching layer on top of the ONTAP S3 store -to optimize repeated existence checks. It maintains an in-memory cache of object -digests and periodically syncs this cache to disk for persistence. - -The cache helps reduce latency for repeated calls to check object existence, -while still ensuring eventual consistency with the underlying ONTAP S3 store. - -Example JSON5 config: -```json5 -"ontap_s3_existence_cache": { - "index_path": "/path/to/cache/index.json", - "sync_interval_seconds": 300, - "backend": { - "endpoint": "https://ontap-s3-endpoint:443", - "vserver_name": "your-vserver", - "bucket": "your-bucket", - "key_prefix": "test-prefix/" - } -} -``` - -**Type:** [OntapS3ExistenceCacheSpec](#ontaps3existencecachespec) - -### `verify` - -Verify store is used to apply verifications to an underlying -store implementation. It is strongly encouraged to validate -as much data as you can before accepting data from a client, -failing to do so may cause the data in the store to be -populated with invalid data causing all kinds of problems. - -The suggested configuration is to have the CAS validate the -hash and size and the AC validate nothing. - -**Example JSON5 config:** -```json5 -"verify": { - "backend": { - "memory": { - "eviction_policy": { - "max_bytes": "500mb" - } - }, - }, - "verify_size": true, - "verify_hash": true -} -``` - -**Type:** [VerifySpec](#verifyspec) - -### `completeness_checking` - -Completeness checking store verifies if the -output files & folders exist in the CAS before forwarding -the request to the underlying store. -Note: This store should only be used on AC stores. - -**Example JSON5 config:** -```json5 -"completeness_checking": { - "backend": { - "filesystem": { - "content_path": "~/.cache/nativelink/content_path-ac", - "temp_path": "~/.cache/nativelink/tmp_path-ac", - "eviction_policy": { - "max_bytes": "500mb", - } - } - }, - "cas_store": { - "ref_store": { - "name": "CAS_MAIN_STORE" - } - } -} -``` - -**Type:** [CompletenessCheckingSpec](#completenesscheckingspec) - -### `compression` - -A compression store that will compress the data inbound and -outbound. There will be a non-trivial cost to compress and -decompress the data, but in many cases if the final store is -a store that requires network transport and/or storage space -is a concern it is often faster and more efficient to use this -store before those stores. - -**Example JSON5 config:** -```json5 -"compression": { - "compression_algorithm": { - "lz4": {} - }, - "backend": { - "filesystem": { - "content_path": "/tmp/nativelink/data/content_path-cas", - "temp_path": "/tmp/nativelink/data/tmp_path-cas", - "eviction_policy": { - "max_bytes": "2gb", - } - } - } -} -``` - -**Type:** [CompressionSpec](#compressionspec) - -### `dedup` - -A dedup store will take the inputs and run a rolling hash -algorithm on them to slice the input into smaller parts then -run a sha256 algorithm on the slice and if the object doesn't -already exist, upload the slice to the `content_store` using -a new digest of just the slice. Once all parts exist, an -Action-Cache-like digest will be built and uploaded to the -`index_store` which will contain a reference to each -chunk/digest of the uploaded file. Downloading a request will -first grab the index from the `index_store`, and forward the -download content of each chunk as if it were one file. - -This store is exceptionally good when the following conditions -are met: -* Content is mostly the same (inserts, updates, deletes are ok) -* Content is not compressed or encrypted -* Uploading or downloading from `content_store` is the bottleneck. - -Note: This store pairs well when used with `CompressionSpec` as -the `content_store`, but never put `DedupSpec` as the backend of -`CompressionSpec` as it will negate all the gains. - -Note: When running `.has()` on this store, it will only check -to see if the entry exists in the `index_store` and not check -if the individual chunks exist in the `content_store`. - -**Example JSON5 config:** -```json5 -"dedup": { - "index_store": { - "memory": { - "eviction_policy": { - "max_bytes": "1GB", - } - } - }, - "content_store": { - "compression": { - "compression_algorithm": { - "lz4": {} - }, - "backend": { - "fast_slow": { - "fast": { - "memory": { - "eviction_policy": { - "max_bytes": "500MB", - } - } - }, - "slow": { - "filesystem": { - "content_path": "/tmp/nativelink/data/content_path-content", - "temp_path": "/tmp/nativelink/data/tmp_path-content", - "eviction_policy": { - "max_bytes": "2gb" - } - } - } - } - } - } - } -} -``` - -**Type:** [DedupSpec](#dedupspec) - -### `existence_cache` - -Existence store will wrap around another store and cache calls -to has so that subsequent `has_with_results` calls will be -faster. This is useful for cases when you have a store that -is slow to respond to has calls. -Note: This store should only be used on CAS stores. - -**Example JSON5 config:** -```json5 -"existence_cache": { - "backend": { - "memory": { - "eviction_policy": { - "max_bytes": "500mb", - } - } - }, - // Note this is the existence store policy, not the backend policy - "eviction_policy": { - "max_seconds": 100, - } -} -``` - -**Type:** [ExistenceCacheSpec](#existencecachespec) - -### `fast_slow` - -`FastSlow` store will first try to fetch the data from the `fast` -store and then if it does not exist try the `slow` store. -When the object does exist in the `slow` store, it will copy -the data to the `fast` store while returning the data. -This store should be thought of as a store that "buffers" -the data to the `fast` store. -On uploads it will mirror data to both `fast` and `slow` stores. - -WARNING: If you need data to always exist in the `slow` store -for something like remote execution, be careful because this -store will never check to see if the objects exist in the -`slow` store if it exists in the `fast` store (i.e., it assumes -that if an object exists in the `fast` store it will exist in -the `slow` store). - -***Example JSON5 config:*** -```json5 -"fast_slow": { - "fast": { - "filesystem": { - "content_path": "/tmp/nativelink/data/content_path-index", - "temp_path": "/tmp/nativelink/data/tmp_path-index", - "eviction_policy": { - "max_bytes": "500mb", - } - } - }, - "slow": { - "filesystem": { - "content_path": "/tmp/nativelink/data/content_path-index", - "temp_path": "/tmp/nativelink/data/tmp_path-index", - "eviction_policy": { - "max_bytes": "500mb", - } - } - } -} -``` - -**Type:** [FastSlowSpec](#fastslowspec) - -### `shard` - -Shards the data to multiple stores. This is useful for cases -when you want to distribute the load across multiple stores. -The digest hash is used to determine which store to send the -data to. - -**Example JSON5 config:** -```json5 -"shard": { - "stores": [ - { - "store": { - "memory": { - "eviction_policy": { - "max_bytes": "10mb" - }, - }, - }, - "weight": 1 - }] -} -``` - -**Type:** [ShardSpec](#shardspec) - -### `filesystem` - -Stores the data on the filesystem. This store is designed for -local persistent storage. Restarts of this program should restore -the previous state, meaning anything uploaded will be persistent -as long as the filesystem integrity holds. - -**Example JSON5 config:** -```json5 -"filesystem": { - "content_path": "/tmp/nativelink/data-worker-test/content_path-cas", - "temp_path": "/tmp/nativelink/data-worker-test/tmp_path-cas", - "eviction_policy": { - "max_bytes": "10gb", - } -} -``` - -**Type:** [FilesystemSpec](#filesystemspec) - -### `ref_store` - -Store used to reference a store in the root store manager. -This is useful for cases when you want to share a store in different -nested stores. Example, you may want to share the same memory store -used for the action cache, but use a `FastSlowSpec` and have the fast -store also share the memory store for efficiency. - -**Example JSON5 config:** -```json5 -"ref_store": { - "name": "FS_CONTENT_STORE" -} -``` - -**Type:** [RefSpec](#refspec) - -### `size_partitioning` - -Uses the size field of the digest to separate which store to send the -data. This is useful for cases when you'd like to put small objects -in one store and large objects in another store. This should only be -used if the size field is the real size of the content, in other -words, don't use on AC (Action Cache) stores. Any store where you can -safely use `VerifySpec.verify_size = true`, this store should be safe -to use (i.e., CAS stores). - -**Example JSON5 config:** -```json5 -"size_partitioning": { - "size": "128mib", - "lower_store": { - "memory": { - "eviction_policy": { - "max_bytes": "${NATIVELINK_CAS_MEMORY_CONTENT_LIMIT:-100mb}" - } - } - }, - "upper_store": { - /// This store discards data larger than 128mib. - "noop": {} - } -} -``` - -**Type:** [SizePartitioningSpec](#sizepartitioningspec) - -### `grpc` - -This store will pass-through calls to another GRPC store. This store -is not designed to be used as a sub-store of another store, but it -does satisfy the interface and will likely work. - -One major GOTCHA is that some stores use a special function on this -store to get the size of the underlying object, which is only reliable -when this store is serving the a CAS store, not an AC store. If using -this store directly without being a child of any store there are no -side effects and is the most efficient way to use it. - -**Example JSON5 config:** -```json5 -"grpc": { - "instance_name": "main", - "endpoints": [ - {"address": "grpc://${CAS_ENDPOINT:-127.0.0.1}:50051"} - ], - "connections_per_endpoint": "5", - "rpc_timeout_s": "5m", - "store_type": "ac", - // Static headers attached to every outgoing request to the upstream - // remote cache. Useful for fixed service-account credentials. - "headers": { - "authorization": "Bearer my-static-token" - }, - // Header names to copy from the inbound client request and forward to - // the upstream remote cache. Use this to pass through dynamic - // credentials such as a JWT sent by the build client. - "forward_headers": ["authorization", "x-custom-token"] -} -``` - -**Type:** [GrpcSpec](#grpcspec) - -### `redis_store` - -Stores data in any stores compatible with Redis APIs. - -Pairs well with `SizePartitioning` and/or `FastSlow` stores. -Ideal for accepting small object sizes as most Redis store -services have a max file upload of between 256Mb-512Mb. - -**Example JSON5 config:** -```json5 -"redis_store": { - "addresses": [ - "redis://127.0.0.1:6379/", - ], - "max_client_permits": 1000, -} -``` - -**Type:** [RedisSpec](#redisspec) - -### `noop` - -Noop store is a store that sends streams into the void and all data -retrieval will return 404 (`NotFound`). This can be useful for cases -where you may need to partition your data and part of your data needs -to be discarded. - -**Example JSON5 config:** -```json5 -"noop": {} -``` - -**Type:** [NoopSpec](#noopspec) - -### `experimental_mongo` - -Experimental `MongoDB` store implementation. - -This store uses `MongoDB` as a backend for storing data. It supports -both CAS (Content Addressable Storage) and scheduler data with -optional change streams for real-time updates. - -**Example JSON5 config:** -```json5 -"experimental_mongo": { - "connection_string": "mongodb://localhost:27017", - "database": "nativelink", - "cas_collection": "cas", - "key_prefix": "cas:", - "read_chunk_size": 65536, - "max_concurrent_uploads": 10, - "enable_change_streams": false, - "max_requests": "100" -} -``` - -**Type:** [ExperimentalMongoSpec](#experimentalmongospec) - ## CompressionAlgorithm Plus exactly one of the following variants (the key selects the variant): @@ -1574,7 +1017,7 @@ Configuration for an individual shard of the store. | Field | Type | Required | Default | Description | | --- | --- | --- | --- | --- | -| `store` | [StoreSpec](#storespec) | Yes | — | Store to shard the data to. | +| `store` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | Store to shard the data to. | | `weight` | integer (uint32) | — | 1 | The weight of the store. This is used to determine how much data should be sent to the store. The actual percentage is the sum of all the store's weights divided by the individual store's weight. | ## GrpcEndpoint diff --git a/web/apps/docs/content/docs/reference/nativelink-config/v1.3.0.mdx b/web/apps/docs/content/docs/reference/nativelink-config/v1.3.0.mdx index 7cb96983f..f4840aff0 100644 --- a/web/apps/docs/content/docs/reference/nativelink-config/v1.3.0.mdx +++ b/web/apps/docs/content/docs/reference/nativelink-config/v1.3.0.mdx @@ -127,8 +127,8 @@ It supports the following backends: } ``` -4. **`NetApp` ONTAP S3** - `NetApp` ONTAP S3 store will use ONTAP's S3-compatible storage as a backend +4. **NetApp ONTAP S3:** + NetApp ONTAP S3 store will use ONTAP's S3-compatible storage as a backend to store files. This store is specifically configured for ONTAP's S3 requirements including custom TLS configuration, credentials management, and proper vserver configuration. @@ -677,7 +677,7 @@ See [`experimental_cloud_object_store`](#experimental_cloud_object_store-1) for | Field | Type | Required | Default | Description | | --- | --- | --- | --- | --- | -| `backend` | [StoreSpec](#storespec) | Yes | — | The underlying store wrap around. All content will first flow through self before forwarding to backend. In the event there is an error detected in self, the connection to the backend will be terminated, and early termination should always cause updates to fail on the backend. | +| `backend` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | The underlying store wrap around. All content will first flow through self before forwarding to backend. In the event there is an error detected in self, the connection to the backend will be terminated, and early termination should always cause updates to fail on the backend. | | `verify_size` | boolean | — | `false` | If set the store will verify the size of the data before accepting an upload of data. | | `verify_hash` | boolean | — | `false` | If the data should be hashed and verify that the key matches the computed hash. The hash function is automatically determined based request and if not set will use the global default. | @@ -685,22 +685,22 @@ See [`experimental_cloud_object_store`](#experimental_cloud_object_store-1) for | Field | Type | Required | Default | Description | | --- | --- | --- | --- | --- | -| `backend` | [StoreSpec](#storespec) | Yes | — | The underlying store that will have it's results validated before sending to client. | -| `cas_store` | [StoreSpec](#storespec) | Yes | — | When a request is made, the results are decoded and all output digests/files are verified to exist in this CAS store before returning success. | +| `backend` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | The underlying store that will have it's results validated before sending to client. | +| `cas_store` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | When a request is made, the results are decoded and all output digests/files are verified to exist in this CAS store before returning success. | ## CompressionSpec | Field | Type | Required | Default | Description | | --- | --- | --- | --- | --- | -| `backend` | [StoreSpec](#storespec) | Yes | — | The underlying store wrap around. All content will first flow through self before forwarding to backend. In the event there is an error detected in self, the connection to the backend will be terminated, and early termination should always cause updates to fail on the backend. | +| `backend` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | The underlying store wrap around. All content will first flow through self before forwarding to backend. In the event there is an error detected in self, the connection to the backend will be terminated, and early termination should always cause updates to fail on the backend. | | `compression_algorithm` | [CompressionAlgorithm](#compressionalgorithm) | Yes | — | The compression algorithm to use. | ## DedupSpec | Field | Type | Required | Default | Description | | --- | --- | --- | --- | --- | -| `index_store` | [StoreSpec](#storespec) | Yes | — | Store used to store the index of each dedup slice. This store should generally be fast and small. | -| `content_store` | [StoreSpec](#storespec) | Yes | — | The store where the individual chunks will be uploaded. This store should generally be the slower & larger store. | +| `index_store` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | Store used to store the index of each dedup slice. This store should generally be fast and small. | +| `content_store` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | The store where the individual chunks will be uploaded. This store should generally be the slower & larger store. | | `min_size` | integer (uint32) | — | 65536 (64k) | Minimum size that a chunk will be when slicing up the content. Note: This setting can be increased to improve performance because it will actually not check this number of bytes when deciding where to partition the data. | | `normal_size` | integer (uint32) | — | 262144 (256k) | A best-effort attempt will be made to keep the average size of the chunks to this number. It is not a guarantee, but a slight attempt will be made. | | `max_size` | integer (uint32) | — | 524288 (512k) | Maximum size a chunk is allowed to be. | @@ -710,16 +710,16 @@ See [`experimental_cloud_object_store`](#experimental_cloud_object_store-1) for | Field | Type | Required | Default | Description | | --- | --- | --- | --- | --- | -| `backend` | [StoreSpec](#storespec) | Yes | — | The underlying store wrap around. All content will first flow through self before forwarding to backend. In the event there is an error detected in self, the connection to the backend will be terminated, and early termination should always cause updates to fail on the backend. | +| `backend` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | The underlying store wrap around. All content will first flow through self before forwarding to backend. In the event there is an error detected in self, the connection to the backend will be terminated, and early termination should always cause updates to fail on the backend. | | `eviction_policy` | [EvictionPolicy](#evictionpolicy) | — | — | Policy used to evict items out of the store. Failure to set this value will cause items to never be removed from the store causing infinite memory usage. | ## FastSlowSpec | Field | Type | Required | Default | Description | | --- | --- | --- | --- | --- | -| `fast` | [StoreSpec](#storespec) | Yes | — | Fast store that will be attempted to be contacted before reaching out to the `slow` store. | +| `fast` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | Fast store that will be attempted to be contacted before reaching out to the `slow` store. | | `fast_direction` | [StoreDirection](#storedirection) | — | `"both"` | How to handle the fast store. This can be useful to set to Get for worker nodes such that results are persisted to the slow store only. | -| `slow` | [StoreSpec](#storespec) | Yes | — | If the object does not exist in the `fast` store it will try to get it from this store. | +| `slow` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | If the object does not exist in the `fast` store it will try to get it from this store. | | `slow_direction` | [StoreDirection](#storedirection) | — | `"both"` | How to handle the slow store. This can be useful if creating a diode and you wish to have an upstream read only store. | ## ShardSpec @@ -750,8 +750,8 @@ See [`experimental_cloud_object_store`](#experimental_cloud_object_store-1) for | Field | Type | Required | Default | Description | | --- | --- | --- | --- | --- | | `size` | integer (uint64) | Yes | — | Size to partition the data on. | -| `lower_store` | [StoreSpec](#storespec) | Yes | — | Store to send data when object is < (less than) size. | -| `upper_store` | [StoreSpec](#storespec) | Yes | — | Store to send data when object is >= (less than eq) size. | +| `lower_store` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | Store to send data when object is < (less than) size. | +| `upper_store` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | Store to send data when object is >= (less than eq) size. | ## GrpcSpec @@ -987,563 +987,6 @@ The total delay is additive, so this example produces 9.525 to 15.875 s of total | `insecure_allow_http` | boolean | — | false | Allow unencrypted HTTP connections. Only use this for local testing. | | `disable_http2` | boolean | — | false | Disable HTTP/2 connections and only use HTTP/1.1. Default client configuration will have HTTP/1.1 and HTTP/2 enabled for connection schemes. HTTP/2 should be disabled if environments have poor support or performance related to HTTP/2. Safe to keep default unless underlying network environment, S3, or GCS API servers specify otherwise. | -## StoreSpec - -Plus exactly one of the following variants (the key selects the variant): - -### `memory` - -Memory store will store all data in a hash map in memory. - -**Example JSON5 config:** -```json5 -"memory": { - "eviction_policy": { - "max_bytes": "10mb", - } -} -``` - -**Type:** [MemorySpec](#memoryspec) - -### `experimental_cloud_object_store` - -A generic blob store that will store files on the cloud -provider. This configuration will never delete files, so you are -responsible for purging old files in other ways. -It supports the following backends: - -1. **Amazon S3:** - S3 store will use Amazon's S3 service as a backend to store - the files. This configuration can be used to share files - across multiple instances. Uses system certificates for TLS - verification via `rustls-platform-verifier`. - - **Example JSON5 config:** - ```json5 - "experimental_cloud_object_store": { - "provider": "aws", - "region": "eu-north-1", - "bucket": "crossplane-bucket-af79aeca9", - "key_prefix": "test-prefix-index/", - "retry": { - "max_retries": 6, - "delay": 0.3, - "jitter": 0.5 - }, - "multipart_max_concurrent_uploads": 10 - } - ``` - -2. **Google Cloud Storage:** - GCS store uses Google's GCS service as a backend to store - the files. This configuration can be used to share files - across multiple instances. - - **Example JSON5 config:** - ```json5 - "experimental_cloud_object_store": { - "provider": "gcs", - "bucket": "test-bucket", - "key_prefix": "test-prefix-index/", - "retry": { - "max_retries": 6, - "delay": 0.3, - "jitter": 0.5 - }, - "multipart_max_concurrent_uploads": 10 - } - ``` - -3. **Azure Blob Store:** - Azure Blob store will use Microsoft's Azure Blob service as a - backend to store the files. This configuration can be used to - share files across multiple instances. - - **Example JSON5 config:** - ```json5 - "experimental_cloud_object_store": { - "provider": "azure", - "account_name": "cloudshell1393657559", - "container": "simple-test-container", - "key_prefix": "folder/", - "retry": { - "max_retries": 6, - "delay": 0.3, - "jitter": 0.5 - }, - "multipart_max_concurrent_uploads": 10 - } - ``` - -4. **`NetApp` ONTAP S3** - `NetApp` ONTAP S3 store will use ONTAP's S3-compatible storage as a backend - to store files. This store is specifically configured for ONTAP's S3 requirements - including custom TLS configuration, credentials management, and proper vserver - configuration. - - This store uses AWS environment variables for credentials: - - `AWS_ACCESS_KEY_ID` - - `AWS_SECRET_ACCESS_KEY` - - `AWS_DEFAULT_REGION` - - **Example JSON5 config:** - ```json5 - "experimental_cloud_object_store": { - "provider": "ontap", - "endpoint": "https://ontap-s3-endpoint:443", - "vserver_name": "your-vserver", - "bucket": "your-bucket", - "root_certificates": "/path/to/certs.pem", // Optional - "key_prefix": "test-prefix/", // Optional - "retry": { - "max_retries": 6, - "delay": 0.3, - "jitter": 0.5 - }, - "multipart_max_concurrent_uploads": 10 - } - ``` - -**Type:** [ExperimentalCloudObjectSpec](#experimentalcloudobjectspec) - -### `ontap_s3_existence_cache` - -ONTAP S3 Existence Cache provides a caching layer on top of the ONTAP S3 store -to optimize repeated existence checks. It maintains an in-memory cache of object -digests and periodically syncs this cache to disk for persistence. - -The cache helps reduce latency for repeated calls to check object existence, -while still ensuring eventual consistency with the underlying ONTAP S3 store. - -Example JSON5 config: -```json5 -"ontap_s3_existence_cache": { - "index_path": "/path/to/cache/index.json", - "sync_interval_seconds": 300, - "backend": { - "endpoint": "https://ontap-s3-endpoint:443", - "vserver_name": "your-vserver", - "bucket": "your-bucket", - "key_prefix": "test-prefix/" - } -} -``` - -**Type:** [OntapS3ExistenceCacheSpec](#ontaps3existencecachespec) - -### `verify` - -Verify store is used to apply verifications to an underlying -store implementation. It is strongly encouraged to validate -as much data as you can before accepting data from a client, -failing to do so may cause the data in the store to be -populated with invalid data causing all kinds of problems. - -The suggested configuration is to have the CAS validate the -hash and size and the AC validate nothing. - -**Example JSON5 config:** -```json5 -"verify": { - "backend": { - "memory": { - "eviction_policy": { - "max_bytes": "500mb" - } - }, - }, - "verify_size": true, - "verify_hash": true -} -``` - -**Type:** [VerifySpec](#verifyspec) - -### `completeness_checking` - -Completeness checking store verifies if the -output files & folders exist in the CAS before forwarding -the request to the underlying store. -Note: This store should only be used on AC stores. - -**Example JSON5 config:** -```json5 -"completeness_checking": { - "backend": { - "filesystem": { - "content_path": "~/.cache/nativelink/content_path-ac", - "temp_path": "~/.cache/nativelink/tmp_path-ac", - "eviction_policy": { - "max_bytes": "500mb", - } - } - }, - "cas_store": { - "ref_store": { - "name": "CAS_MAIN_STORE" - } - } -} -``` - -**Type:** [CompletenessCheckingSpec](#completenesscheckingspec) - -### `compression` - -A compression store that will compress the data inbound and -outbound. There will be a non-trivial cost to compress and -decompress the data, but in many cases if the final store is -a store that requires network transport and/or storage space -is a concern it is often faster and more efficient to use this -store before those stores. - -**Example JSON5 config:** -```json5 -"compression": { - "compression_algorithm": { - "lz4": {} - }, - "backend": { - "filesystem": { - "content_path": "/tmp/nativelink/data/content_path-cas", - "temp_path": "/tmp/nativelink/data/tmp_path-cas", - "eviction_policy": { - "max_bytes": "2gb", - } - } - } -} -``` - -**Type:** [CompressionSpec](#compressionspec) - -### `dedup` - -A dedup store will take the inputs and run a rolling hash -algorithm on them to slice the input into smaller parts then -run a sha256 algorithm on the slice and if the object doesn't -already exist, upload the slice to the `content_store` using -a new digest of just the slice. Once all parts exist, an -Action-Cache-like digest will be built and uploaded to the -`index_store` which will contain a reference to each -chunk/digest of the uploaded file. Downloading a request will -first grab the index from the `index_store`, and forward the -download content of each chunk as if it were one file. - -This store is exceptionally good when the following conditions -are met: -* Content is mostly the same (inserts, updates, deletes are ok) -* Content is not compressed or encrypted -* Uploading or downloading from `content_store` is the bottleneck. - -Note: This store pairs well when used with `CompressionSpec` as -the `content_store`, but never put `DedupSpec` as the backend of -`CompressionSpec` as it will negate all the gains. - -Note: When running `.has()` on this store, it will only check -to see if the entry exists in the `index_store` and not check -if the individual chunks exist in the `content_store`. - -**Example JSON5 config:** -```json5 -"dedup": { - "index_store": { - "memory": { - "eviction_policy": { - "max_bytes": "1GB", - } - } - }, - "content_store": { - "compression": { - "compression_algorithm": { - "lz4": {} - }, - "backend": { - "fast_slow": { - "fast": { - "memory": { - "eviction_policy": { - "max_bytes": "500MB", - } - } - }, - "slow": { - "filesystem": { - "content_path": "/tmp/nativelink/data/content_path-content", - "temp_path": "/tmp/nativelink/data/tmp_path-content", - "eviction_policy": { - "max_bytes": "2gb" - } - } - } - } - } - } - } -} -``` - -**Type:** [DedupSpec](#dedupspec) - -### `existence_cache` - -Existence store will wrap around another store and cache calls -to has so that subsequent `has_with_results` calls will be -faster. This is useful for cases when you have a store that -is slow to respond to has calls. -Note: This store should only be used on CAS stores. - -**Example JSON5 config:** -```json5 -"existence_cache": { - "backend": { - "memory": { - "eviction_policy": { - "max_bytes": "500mb", - } - } - }, - // Note this is the existence store policy, not the backend policy - "eviction_policy": { - "max_seconds": 100, - } -} -``` - -**Type:** [ExistenceCacheSpec](#existencecachespec) - -### `fast_slow` - -`FastSlow` store will first try to fetch the data from the `fast` -store and then if it does not exist try the `slow` store. -When the object does exist in the `slow` store, it will copy -the data to the `fast` store while returning the data. -This store should be thought of as a store that "buffers" -the data to the `fast` store. -On uploads it will mirror data to both `fast` and `slow` stores. - -WARNING: If you need data to always exist in the `slow` store -for something like remote execution, be careful because this -store will never check to see if the objects exist in the -`slow` store if it exists in the `fast` store (i.e., it assumes -that if an object exists in the `fast` store it will exist in -the `slow` store). - -***Example JSON5 config:*** -```json5 -"fast_slow": { - "fast": { - "filesystem": { - "content_path": "/tmp/nativelink/data/content_path-index", - "temp_path": "/tmp/nativelink/data/tmp_path-index", - "eviction_policy": { - "max_bytes": "500mb", - } - } - }, - "slow": { - "filesystem": { - "content_path": "/tmp/nativelink/data/content_path-index", - "temp_path": "/tmp/nativelink/data/tmp_path-index", - "eviction_policy": { - "max_bytes": "500mb", - } - } - } -} -``` - -**Type:** [FastSlowSpec](#fastslowspec) - -### `shard` - -Shards the data to multiple stores. This is useful for cases -when you want to distribute the load across multiple stores. -The digest hash is used to determine which store to send the -data to. - -**Example JSON5 config:** -```json5 -"shard": { - "stores": [ - { - "store": { - "memory": { - "eviction_policy": { - "max_bytes": "10mb" - }, - }, - }, - "weight": 1 - }] -} -``` - -**Type:** [ShardSpec](#shardspec) - -### `filesystem` - -Stores the data on the filesystem. This store is designed for -local persistent storage. Restarts of this program should restore -the previous state, meaning anything uploaded will be persistent -as long as the filesystem integrity holds. - -**Example JSON5 config:** -```json5 -"filesystem": { - "content_path": "/tmp/nativelink/data-worker-test/content_path-cas", - "temp_path": "/tmp/nativelink/data-worker-test/tmp_path-cas", - "eviction_policy": { - "max_bytes": "10gb", - } -} -``` - -**Type:** [FilesystemSpec](#filesystemspec) - -### `ref_store` - -Store used to reference a store in the root store manager. -This is useful for cases when you want to share a store in different -nested stores. Example, you may want to share the same memory store -used for the action cache, but use a `FastSlowSpec` and have the fast -store also share the memory store for efficiency. - -**Example JSON5 config:** -```json5 -"ref_store": { - "name": "FS_CONTENT_STORE" -} -``` - -**Type:** [RefSpec](#refspec) - -### `size_partitioning` - -Uses the size field of the digest to separate which store to send the -data. This is useful for cases when you'd like to put small objects -in one store and large objects in another store. This should only be -used if the size field is the real size of the content, in other -words, don't use on AC (Action Cache) stores. Any store where you can -safely use `VerifySpec.verify_size = true`, this store should be safe -to use (i.e., CAS stores). - -**Example JSON5 config:** -```json5 -"size_partitioning": { - "size": "128mib", - "lower_store": { - "memory": { - "eviction_policy": { - "max_bytes": "${NATIVELINK_CAS_MEMORY_CONTENT_LIMIT:-100mb}" - } - } - }, - "upper_store": { - /// This store discards data larger than 128mib. - "noop": {} - } -} -``` - -**Type:** [SizePartitioningSpec](#sizepartitioningspec) - -### `grpc` - -This store will pass-through calls to another GRPC store. This store -is not designed to be used as a sub-store of another store, but it -does satisfy the interface and will likely work. - -One major GOTCHA is that some stores use a special function on this -store to get the size of the underlying object, which is only reliable -when this store is serving the a CAS store, not an AC store. If using -this store directly without being a child of any store there are no -side effects and is the most efficient way to use it. - -**Example JSON5 config:** -```json5 -"grpc": { - "instance_name": "main", - "endpoints": [ - {"address": "grpc://${CAS_ENDPOINT:-127.0.0.1}:50051"} - ], - "connections_per_endpoint": "5", - "rpc_timeout_s": "5m", - "store_type": "ac", - // Static headers attached to every outgoing request to the upstream - // remote cache. Useful for fixed service-account credentials. - "headers": { - "authorization": "Bearer my-static-token" - }, - // Header names to copy from the inbound client request and forward to - // the upstream remote cache. Use this to pass through dynamic - // credentials such as a JWT sent by the build client. - "forward_headers": ["authorization", "x-custom-token"] -} -``` - -**Type:** [GrpcSpec](#grpcspec) - -### `redis_store` - -Stores data in any stores compatible with Redis APIs. - -Pairs well with `SizePartitioning` and/or `FastSlow` stores. -Ideal for accepting small object sizes as most Redis store -services have a max file upload of between 256Mb-512Mb. - -**Example JSON5 config:** -```json5 -"redis_store": { - "addresses": [ - "redis://127.0.0.1:6379/", - ], - "max_client_permits": 1000, -} -``` - -**Type:** [RedisSpec](#redisspec) - -### `noop` - -Noop store is a store that sends streams into the void and all data -retrieval will return 404 (`NotFound`). This can be useful for cases -where you may need to partition your data and part of your data needs -to be discarded. - -**Example JSON5 config:** -```json5 -"noop": {} -``` - -**Type:** [NoopSpec](#noopspec) - -### `experimental_mongo` - -Experimental `MongoDB` store implementation. - -This store uses `MongoDB` as a backend for storing data. It supports -both CAS (Content Addressable Storage) and scheduler data with -optional change streams for real-time updates. - -**Example JSON5 config:** -```json5 -"experimental_mongo": { - "connection_string": "mongodb://localhost:27017", - "database": "nativelink", - "cas_collection": "cas", - "key_prefix": "cas:", - "read_chunk_size": 65536, - "max_concurrent_uploads": 10, - "enable_change_streams": false, - "max_requests": "100" -} -``` - -**Type:** [ExperimentalMongoSpec](#experimentalmongospec) - ## CompressionAlgorithm Plus exactly one of the following variants (the key selects the variant): @@ -1575,7 +1018,7 @@ Configuration for an individual shard of the store. | Field | Type | Required | Default | Description | | --- | --- | --- | --- | --- | -| `store` | [StoreSpec](#storespec) | Yes | — | Store to shard the data to. | +| `store` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | Store to shard the data to. | | `weight` | integer (uint32) | — | 1 | The weight of the store. This is used to determine how much data should be sent to the store. The actual percentage is the sum of all the store's weights divided by the individual store's weight. | ## GrpcEndpoint diff --git a/web/apps/docs/content/docs/reference/nativelink-config/v1.3.1.mdx b/web/apps/docs/content/docs/reference/nativelink-config/v1.3.1.mdx index 24c1f5891..2ccf26b6a 100644 --- a/web/apps/docs/content/docs/reference/nativelink-config/v1.3.1.mdx +++ b/web/apps/docs/content/docs/reference/nativelink-config/v1.3.1.mdx @@ -127,8 +127,8 @@ It supports the following backends: } ``` -4. **`NetApp` ONTAP S3** - `NetApp` ONTAP S3 store will use ONTAP's S3-compatible storage as a backend +4. **NetApp ONTAP S3:** + NetApp ONTAP S3 store will use ONTAP's S3-compatible storage as a backend to store files. This store is specifically configured for ONTAP's S3 requirements including custom TLS configuration, credentials management, and proper vserver configuration. @@ -677,7 +677,7 @@ See [`experimental_cloud_object_store`](#experimental_cloud_object_store-1) for | Field | Type | Required | Default | Description | | --- | --- | --- | --- | --- | -| `backend` | [StoreSpec](#storespec) | Yes | — | The underlying store wrap around. All content will first flow through self before forwarding to backend. In the event there is an error detected in self, the connection to the backend will be terminated, and early termination should always cause updates to fail on the backend. | +| `backend` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | The underlying store wrap around. All content will first flow through self before forwarding to backend. In the event there is an error detected in self, the connection to the backend will be terminated, and early termination should always cause updates to fail on the backend. | | `verify_size` | boolean | — | `false` | If set the store will verify the size of the data before accepting an upload of data. | | `verify_hash` | boolean | — | `false` | If the data should be hashed and verify that the key matches the computed hash. The hash function is automatically determined based request and if not set will use the global default. | @@ -685,22 +685,22 @@ See [`experimental_cloud_object_store`](#experimental_cloud_object_store-1) for | Field | Type | Required | Default | Description | | --- | --- | --- | --- | --- | -| `backend` | [StoreSpec](#storespec) | Yes | — | The underlying store that will have it's results validated before sending to client. | -| `cas_store` | [StoreSpec](#storespec) | Yes | — | When a request is made, the results are decoded and all output digests/files are verified to exist in this CAS store before returning success. | +| `backend` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | The underlying store that will have it's results validated before sending to client. | +| `cas_store` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | When a request is made, the results are decoded and all output digests/files are verified to exist in this CAS store before returning success. | ## CompressionSpec | Field | Type | Required | Default | Description | | --- | --- | --- | --- | --- | -| `backend` | [StoreSpec](#storespec) | Yes | — | The underlying store wrap around. All content will first flow through self before forwarding to backend. In the event there is an error detected in self, the connection to the backend will be terminated, and early termination should always cause updates to fail on the backend. | +| `backend` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | The underlying store wrap around. All content will first flow through self before forwarding to backend. In the event there is an error detected in self, the connection to the backend will be terminated, and early termination should always cause updates to fail on the backend. | | `compression_algorithm` | [CompressionAlgorithm](#compressionalgorithm) | Yes | — | The compression algorithm to use. | ## DedupSpec | Field | Type | Required | Default | Description | | --- | --- | --- | --- | --- | -| `index_store` | [StoreSpec](#storespec) | Yes | — | Store used to store the index of each dedup slice. This store should generally be fast and small. | -| `content_store` | [StoreSpec](#storespec) | Yes | — | The store where the individual chunks will be uploaded. This store should generally be the slower & larger store. | +| `index_store` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | Store used to store the index of each dedup slice. This store should generally be fast and small. | +| `content_store` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | The store where the individual chunks will be uploaded. This store should generally be the slower & larger store. | | `min_size` | integer (uint32) | — | 65536 (64k) | Minimum size that a chunk will be when slicing up the content. Note: This setting can be increased to improve performance because it will actually not check this number of bytes when deciding where to partition the data. | | `normal_size` | integer (uint32) | — | 262144 (256k) | A best-effort attempt will be made to keep the average size of the chunks to this number. It is not a guarantee, but a slight attempt will be made. | | `max_size` | integer (uint32) | — | 524288 (512k) | Maximum size a chunk is allowed to be. | @@ -710,16 +710,16 @@ See [`experimental_cloud_object_store`](#experimental_cloud_object_store-1) for | Field | Type | Required | Default | Description | | --- | --- | --- | --- | --- | -| `backend` | [StoreSpec](#storespec) | Yes | — | The underlying store wrap around. All content will first flow through self before forwarding to backend. In the event there is an error detected in self, the connection to the backend will be terminated, and early termination should always cause updates to fail on the backend. | +| `backend` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | The underlying store wrap around. All content will first flow through self before forwarding to backend. In the event there is an error detected in self, the connection to the backend will be terminated, and early termination should always cause updates to fail on the backend. | | `eviction_policy` | [EvictionPolicy](#evictionpolicy) | — | — | Policy used to evict items out of the store. Failure to set this value will cause items to never be removed from the store causing infinite memory usage. | ## FastSlowSpec | Field | Type | Required | Default | Description | | --- | --- | --- | --- | --- | -| `fast` | [StoreSpec](#storespec) | Yes | — | Fast store that will be attempted to be contacted before reaching out to the `slow` store. | +| `fast` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | Fast store that will be attempted to be contacted before reaching out to the `slow` store. | | `fast_direction` | [StoreDirection](#storedirection) | — | `"both"` | How to handle the fast store. This can be useful to set to Get for worker nodes such that results are persisted to the slow store only. | -| `slow` | [StoreSpec](#storespec) | Yes | — | If the object does not exist in the `fast` store it will try to get it from this store. | +| `slow` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | If the object does not exist in the `fast` store it will try to get it from this store. | | `slow_direction` | [StoreDirection](#storedirection) | — | `"both"` | How to handle the slow store. This can be useful if creating a diode and you wish to have an upstream read only store. | ## ShardSpec @@ -750,8 +750,8 @@ See [`experimental_cloud_object_store`](#experimental_cloud_object_store-1) for | Field | Type | Required | Default | Description | | --- | --- | --- | --- | --- | | `size` | integer (uint64) | Yes | — | Size to partition the data on. | -| `lower_store` | [StoreSpec](#storespec) | Yes | — | Store to send data when object is < (less than) size. | -| `upper_store` | [StoreSpec](#storespec) | Yes | — | Store to send data when object is >= (less than eq) size. | +| `lower_store` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | Store to send data when object is < (less than) size. | +| `upper_store` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | Store to send data when object is >= (less than eq) size. | ## GrpcSpec @@ -987,563 +987,6 @@ The total delay is additive, so this example produces 9.525 to 15.875 s of total | `insecure_allow_http` | boolean | — | false | Allow unencrypted HTTP connections. Only use this for local testing. | | `disable_http2` | boolean | — | false | Disable HTTP/2 connections and only use HTTP/1.1. Default client configuration will have HTTP/1.1 and HTTP/2 enabled for connection schemes. HTTP/2 should be disabled if environments have poor support or performance related to HTTP/2. Safe to keep default unless underlying network environment, S3, or GCS API servers specify otherwise. | -## StoreSpec - -Plus exactly one of the following variants (the key selects the variant): - -### `memory` - -Memory store will store all data in a hash map in memory. - -**Example JSON5 config:** -```json5 -"memory": { - "eviction_policy": { - "max_bytes": "10mb", - } -} -``` - -**Type:** [MemorySpec](#memoryspec) - -### `experimental_cloud_object_store` - -A generic blob store that will store files on the cloud -provider. This configuration will never delete files, so you are -responsible for purging old files in other ways. -It supports the following backends: - -1. **Amazon S3:** - S3 store will use Amazon's S3 service as a backend to store - the files. This configuration can be used to share files - across multiple instances. Uses system certificates for TLS - verification via `rustls-platform-verifier`. - - **Example JSON5 config:** - ```json5 - "experimental_cloud_object_store": { - "provider": "aws", - "region": "eu-north-1", - "bucket": "crossplane-bucket-af79aeca9", - "key_prefix": "test-prefix-index/", - "retry": { - "max_retries": 6, - "delay": 0.3, - "jitter": 0.5 - }, - "multipart_max_concurrent_uploads": 10 - } - ``` - -2. **Google Cloud Storage:** - GCS store uses Google's GCS service as a backend to store - the files. This configuration can be used to share files - across multiple instances. - - **Example JSON5 config:** - ```json5 - "experimental_cloud_object_store": { - "provider": "gcs", - "bucket": "test-bucket", - "key_prefix": "test-prefix-index/", - "retry": { - "max_retries": 6, - "delay": 0.3, - "jitter": 0.5 - }, - "multipart_max_concurrent_uploads": 10 - } - ``` - -3. **Azure Blob Store:** - Azure Blob store will use Microsoft's Azure Blob service as a - backend to store the files. This configuration can be used to - share files across multiple instances. - - **Example JSON5 config:** - ```json5 - "experimental_cloud_object_store": { - "provider": "azure", - "account_name": "cloudshell1393657559", - "container": "simple-test-container", - "key_prefix": "folder/", - "retry": { - "max_retries": 6, - "delay": 0.3, - "jitter": 0.5 - }, - "multipart_max_concurrent_uploads": 10 - } - ``` - -4. **`NetApp` ONTAP S3** - `NetApp` ONTAP S3 store will use ONTAP's S3-compatible storage as a backend - to store files. This store is specifically configured for ONTAP's S3 requirements - including custom TLS configuration, credentials management, and proper vserver - configuration. - - This store uses AWS environment variables for credentials: - - `AWS_ACCESS_KEY_ID` - - `AWS_SECRET_ACCESS_KEY` - - `AWS_DEFAULT_REGION` - - **Example JSON5 config:** - ```json5 - "experimental_cloud_object_store": { - "provider": "ontap", - "endpoint": "https://ontap-s3-endpoint:443", - "vserver_name": "your-vserver", - "bucket": "your-bucket", - "root_certificates": "/path/to/certs.pem", // Optional - "key_prefix": "test-prefix/", // Optional - "retry": { - "max_retries": 6, - "delay": 0.3, - "jitter": 0.5 - }, - "multipart_max_concurrent_uploads": 10 - } - ``` - -**Type:** [ExperimentalCloudObjectSpec](#experimentalcloudobjectspec) - -### `ontap_s3_existence_cache` - -ONTAP S3 Existence Cache provides a caching layer on top of the ONTAP S3 store -to optimize repeated existence checks. It maintains an in-memory cache of object -digests and periodically syncs this cache to disk for persistence. - -The cache helps reduce latency for repeated calls to check object existence, -while still ensuring eventual consistency with the underlying ONTAP S3 store. - -Example JSON5 config: -```json5 -"ontap_s3_existence_cache": { - "index_path": "/path/to/cache/index.json", - "sync_interval_seconds": 300, - "backend": { - "endpoint": "https://ontap-s3-endpoint:443", - "vserver_name": "your-vserver", - "bucket": "your-bucket", - "key_prefix": "test-prefix/" - } -} -``` - -**Type:** [OntapS3ExistenceCacheSpec](#ontaps3existencecachespec) - -### `verify` - -Verify store is used to apply verifications to an underlying -store implementation. It is strongly encouraged to validate -as much data as you can before accepting data from a client, -failing to do so may cause the data in the store to be -populated with invalid data causing all kinds of problems. - -The suggested configuration is to have the CAS validate the -hash and size and the AC validate nothing. - -**Example JSON5 config:** -```json5 -"verify": { - "backend": { - "memory": { - "eviction_policy": { - "max_bytes": "500mb" - } - }, - }, - "verify_size": true, - "verify_hash": true -} -``` - -**Type:** [VerifySpec](#verifyspec) - -### `completeness_checking` - -Completeness checking store verifies if the -output files & folders exist in the CAS before forwarding -the request to the underlying store. -Note: This store should only be used on AC stores. - -**Example JSON5 config:** -```json5 -"completeness_checking": { - "backend": { - "filesystem": { - "content_path": "~/.cache/nativelink/content_path-ac", - "temp_path": "~/.cache/nativelink/tmp_path-ac", - "eviction_policy": { - "max_bytes": "500mb", - } - } - }, - "cas_store": { - "ref_store": { - "name": "CAS_MAIN_STORE" - } - } -} -``` - -**Type:** [CompletenessCheckingSpec](#completenesscheckingspec) - -### `compression` - -A compression store that will compress the data inbound and -outbound. There will be a non-trivial cost to compress and -decompress the data, but in many cases if the final store is -a store that requires network transport and/or storage space -is a concern it is often faster and more efficient to use this -store before those stores. - -**Example JSON5 config:** -```json5 -"compression": { - "compression_algorithm": { - "lz4": {} - }, - "backend": { - "filesystem": { - "content_path": "/tmp/nativelink/data/content_path-cas", - "temp_path": "/tmp/nativelink/data/tmp_path-cas", - "eviction_policy": { - "max_bytes": "2gb", - } - } - } -} -``` - -**Type:** [CompressionSpec](#compressionspec) - -### `dedup` - -A dedup store will take the inputs and run a rolling hash -algorithm on them to slice the input into smaller parts then -run a sha256 algorithm on the slice and if the object doesn't -already exist, upload the slice to the `content_store` using -a new digest of just the slice. Once all parts exist, an -Action-Cache-like digest will be built and uploaded to the -`index_store` which will contain a reference to each -chunk/digest of the uploaded file. Downloading a request will -first grab the index from the `index_store`, and forward the -download content of each chunk as if it were one file. - -This store is exceptionally good when the following conditions -are met: -* Content is mostly the same (inserts, updates, deletes are ok) -* Content is not compressed or encrypted -* Uploading or downloading from `content_store` is the bottleneck. - -Note: This store pairs well when used with `CompressionSpec` as -the `content_store`, but never put `DedupSpec` as the backend of -`CompressionSpec` as it will negate all the gains. - -Note: When running `.has()` on this store, it will only check -to see if the entry exists in the `index_store` and not check -if the individual chunks exist in the `content_store`. - -**Example JSON5 config:** -```json5 -"dedup": { - "index_store": { - "memory": { - "eviction_policy": { - "max_bytes": "1GB", - } - } - }, - "content_store": { - "compression": { - "compression_algorithm": { - "lz4": {} - }, - "backend": { - "fast_slow": { - "fast": { - "memory": { - "eviction_policy": { - "max_bytes": "500MB", - } - } - }, - "slow": { - "filesystem": { - "content_path": "/tmp/nativelink/data/content_path-content", - "temp_path": "/tmp/nativelink/data/tmp_path-content", - "eviction_policy": { - "max_bytes": "2gb" - } - } - } - } - } - } - } -} -``` - -**Type:** [DedupSpec](#dedupspec) - -### `existence_cache` - -Existence store will wrap around another store and cache calls -to has so that subsequent `has_with_results` calls will be -faster. This is useful for cases when you have a store that -is slow to respond to has calls. -Note: This store should only be used on CAS stores. - -**Example JSON5 config:** -```json5 -"existence_cache": { - "backend": { - "memory": { - "eviction_policy": { - "max_bytes": "500mb", - } - } - }, - // Note this is the existence store policy, not the backend policy - "eviction_policy": { - "max_seconds": 100, - } -} -``` - -**Type:** [ExistenceCacheSpec](#existencecachespec) - -### `fast_slow` - -`FastSlow` store will first try to fetch the data from the `fast` -store and then if it does not exist try the `slow` store. -When the object does exist in the `slow` store, it will copy -the data to the `fast` store while returning the data. -This store should be thought of as a store that "buffers" -the data to the `fast` store. -On uploads it will mirror data to both `fast` and `slow` stores. - -WARNING: If you need data to always exist in the `slow` store -for something like remote execution, be careful because this -store will never check to see if the objects exist in the -`slow` store if it exists in the `fast` store (i.e., it assumes -that if an object exists in the `fast` store it will exist in -the `slow` store). - -***Example JSON5 config:*** -```json5 -"fast_slow": { - "fast": { - "filesystem": { - "content_path": "/tmp/nativelink/data/content_path-index", - "temp_path": "/tmp/nativelink/data/tmp_path-index", - "eviction_policy": { - "max_bytes": "500mb", - } - } - }, - "slow": { - "filesystem": { - "content_path": "/tmp/nativelink/data/content_path-index", - "temp_path": "/tmp/nativelink/data/tmp_path-index", - "eviction_policy": { - "max_bytes": "500mb", - } - } - } -} -``` - -**Type:** [FastSlowSpec](#fastslowspec) - -### `shard` - -Shards the data to multiple stores. This is useful for cases -when you want to distribute the load across multiple stores. -The digest hash is used to determine which store to send the -data to. - -**Example JSON5 config:** -```json5 -"shard": { - "stores": [ - { - "store": { - "memory": { - "eviction_policy": { - "max_bytes": "10mb" - }, - }, - }, - "weight": 1 - }] -} -``` - -**Type:** [ShardSpec](#shardspec) - -### `filesystem` - -Stores the data on the filesystem. This store is designed for -local persistent storage. Restarts of this program should restore -the previous state, meaning anything uploaded will be persistent -as long as the filesystem integrity holds. - -**Example JSON5 config:** -```json5 -"filesystem": { - "content_path": "/tmp/nativelink/data-worker-test/content_path-cas", - "temp_path": "/tmp/nativelink/data-worker-test/tmp_path-cas", - "eviction_policy": { - "max_bytes": "10gb", - } -} -``` - -**Type:** [FilesystemSpec](#filesystemspec) - -### `ref_store` - -Store used to reference a store in the root store manager. -This is useful for cases when you want to share a store in different -nested stores. Example, you may want to share the same memory store -used for the action cache, but use a `FastSlowSpec` and have the fast -store also share the memory store for efficiency. - -**Example JSON5 config:** -```json5 -"ref_store": { - "name": "FS_CONTENT_STORE" -} -``` - -**Type:** [RefSpec](#refspec) - -### `size_partitioning` - -Uses the size field of the digest to separate which store to send the -data. This is useful for cases when you'd like to put small objects -in one store and large objects in another store. This should only be -used if the size field is the real size of the content, in other -words, don't use on AC (Action Cache) stores. Any store where you can -safely use `VerifySpec.verify_size = true`, this store should be safe -to use (i.e., CAS stores). - -**Example JSON5 config:** -```json5 -"size_partitioning": { - "size": "128mib", - "lower_store": { - "memory": { - "eviction_policy": { - "max_bytes": "${NATIVELINK_CAS_MEMORY_CONTENT_LIMIT:-100mb}" - } - } - }, - "upper_store": { - /// This store discards data larger than 128mib. - "noop": {} - } -} -``` - -**Type:** [SizePartitioningSpec](#sizepartitioningspec) - -### `grpc` - -This store will pass-through calls to another GRPC store. This store -is not designed to be used as a sub-store of another store, but it -does satisfy the interface and will likely work. - -One major GOTCHA is that some stores use a special function on this -store to get the size of the underlying object, which is only reliable -when this store is serving the a CAS store, not an AC store. If using -this store directly without being a child of any store there are no -side effects and is the most efficient way to use it. - -**Example JSON5 config:** -```json5 -"grpc": { - "instance_name": "main", - "endpoints": [ - {"address": "grpc://${CAS_ENDPOINT:-127.0.0.1}:50051"} - ], - "connections_per_endpoint": "5", - "rpc_timeout_s": "5m", - "store_type": "ac", - // Static headers attached to every outgoing request to the upstream - // remote cache. Useful for fixed service-account credentials. - "headers": { - "authorization": "Bearer my-static-token" - }, - // Header names to copy from the inbound client request and forward to - // the upstream remote cache. Use this to pass through dynamic - // credentials such as a JWT sent by the build client. - "forward_headers": ["authorization", "x-custom-token"] -} -``` - -**Type:** [GrpcSpec](#grpcspec) - -### `redis_store` - -Stores data in any stores compatible with Redis APIs. - -Pairs well with `SizePartitioning` and/or `FastSlow` stores. -Ideal for accepting small object sizes as most Redis store -services have a max file upload of between 256Mb-512Mb. - -**Example JSON5 config:** -```json5 -"redis_store": { - "addresses": [ - "redis://127.0.0.1:6379/", - ], - "max_client_permits": 1000, -} -``` - -**Type:** [RedisSpec](#redisspec) - -### `noop` - -Noop store is a store that sends streams into the void and all data -retrieval will return 404 (`NotFound`). This can be useful for cases -where you may need to partition your data and part of your data needs -to be discarded. - -**Example JSON5 config:** -```json5 -"noop": {} -``` - -**Type:** [NoopSpec](#noopspec) - -### `experimental_mongo` - -Experimental `MongoDB` store implementation. - -This store uses `MongoDB` as a backend for storing data. It supports -both CAS (Content Addressable Storage) and scheduler data with -optional change streams for real-time updates. - -**Example JSON5 config:** -```json5 -"experimental_mongo": { - "connection_string": "mongodb://localhost:27017", - "database": "nativelink", - "cas_collection": "cas", - "key_prefix": "cas:", - "read_chunk_size": 65536, - "max_concurrent_uploads": 10, - "enable_change_streams": false, - "max_requests": "100" -} -``` - -**Type:** [ExperimentalMongoSpec](#experimentalmongospec) - ## CompressionAlgorithm Plus exactly one of the following variants (the key selects the variant): @@ -1575,7 +1018,7 @@ Configuration for an individual shard of the store. | Field | Type | Required | Default | Description | | --- | --- | --- | --- | --- | -| `store` | [StoreSpec](#storespec) | Yes | — | Store to shard the data to. | +| `store` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | Store to shard the data to. | | `weight` | integer (uint32) | — | 1 | The weight of the store. This is used to determine how much data should be sent to the store. The actual percentage is the sum of all the store's weights divided by the individual store's weight. | ## GrpcEndpoint diff --git a/web/apps/docs/content/docs/reference/nativelink-config/v1.3.2.mdx b/web/apps/docs/content/docs/reference/nativelink-config/v1.3.2.mdx index 27bf375fa..5a78887b6 100644 --- a/web/apps/docs/content/docs/reference/nativelink-config/v1.3.2.mdx +++ b/web/apps/docs/content/docs/reference/nativelink-config/v1.3.2.mdx @@ -151,8 +151,8 @@ It supports the following backends: } ``` -4. **`NetApp` ONTAP S3** - `NetApp` ONTAP S3 store will use ONTAP's S3-compatible storage as a backend +4. **NetApp ONTAP S3:** + NetApp ONTAP S3 store will use ONTAP's S3-compatible storage as a backend to store files. This store is specifically configured for ONTAP's S3 requirements including custom TLS configuration, credentials management, and proper vserver configuration. @@ -684,7 +684,7 @@ Plus exactly one of the following variants (the key selects the variant): | Field | Type | Required | Default | Description | | --- | --- | --- | --- | --- | | `cache_type` | string | Yes | — | Low-cardinality cache type label for metrics, for example `cas` or `ac`. | -| `backend` | [StoreSpec](#storespec) | Yes | — | Store to wrap with cache operation metrics. | +| `backend` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | Store to wrap with cache operation metrics. | ## MemorySpec @@ -708,7 +708,7 @@ See [`experimental_cloud_object_store`](#experimental_cloud_object_store-1) for | Field | Type | Required | Default | Description | | --- | --- | --- | --- | --- | -| `backend` | [StoreSpec](#storespec) | Yes | — | The underlying store wrap around. All content will first flow through self before forwarding to backend. In the event there is an error detected in self, the connection to the backend will be terminated, and early termination should always cause updates to fail on the backend. | +| `backend` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | The underlying store wrap around. All content will first flow through self before forwarding to backend. In the event there is an error detected in self, the connection to the backend will be terminated, and early termination should always cause updates to fail on the backend. | | `verify_size` | boolean | — | `false` | If set the store will verify the size of the data before accepting an upload of data. | | `verify_hash` | boolean | — | `false` | If the data should be hashed and verify that the key matches the computed hash. The hash function is automatically determined based request and if not set will use the global default. | @@ -716,22 +716,22 @@ See [`experimental_cloud_object_store`](#experimental_cloud_object_store-1) for | Field | Type | Required | Default | Description | | --- | --- | --- | --- | --- | -| `backend` | [StoreSpec](#storespec) | Yes | — | The underlying store that will have it's results validated before sending to client. | -| `cas_store` | [StoreSpec](#storespec) | Yes | — | When a request is made, the results are decoded and all output digests/files are verified to exist in this CAS store before returning success. | +| `backend` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | The underlying store that will have it's results validated before sending to client. | +| `cas_store` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | When a request is made, the results are decoded and all output digests/files are verified to exist in this CAS store before returning success. | ## CompressionSpec | Field | Type | Required | Default | Description | | --- | --- | --- | --- | --- | -| `backend` | [StoreSpec](#storespec) | Yes | — | The underlying store wrap around. All content will first flow through self before forwarding to backend. In the event there is an error detected in self, the connection to the backend will be terminated, and early termination should always cause updates to fail on the backend. | +| `backend` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | The underlying store wrap around. All content will first flow through self before forwarding to backend. In the event there is an error detected in self, the connection to the backend will be terminated, and early termination should always cause updates to fail on the backend. | | `compression_algorithm` | [CompressionAlgorithm](#compressionalgorithm) | Yes | — | The compression algorithm to use. | ## DedupSpec | Field | Type | Required | Default | Description | | --- | --- | --- | --- | --- | -| `index_store` | [StoreSpec](#storespec) | Yes | — | Store used to store the index of each dedup slice. This store should generally be fast and small. | -| `content_store` | [StoreSpec](#storespec) | Yes | — | The store where the individual chunks will be uploaded. This store should generally be the slower & larger store. | +| `index_store` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | Store used to store the index of each dedup slice. This store should generally be fast and small. | +| `content_store` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | The store where the individual chunks will be uploaded. This store should generally be the slower & larger store. | | `min_size` | integer (uint32) | — | 65536 (64k) | Minimum size that a chunk will be when slicing up the content. Note: This setting can be increased to improve performance because it will actually not check this number of bytes when deciding where to partition the data. | | `normal_size` | integer (uint32) | — | 262144 (256k) | A best-effort attempt will be made to keep the average size of the chunks to this number. It is not a guarantee, but a slight attempt will be made. | | `max_size` | integer (uint32) | — | 524288 (512k) | Maximum size a chunk is allowed to be. | @@ -741,16 +741,16 @@ See [`experimental_cloud_object_store`](#experimental_cloud_object_store-1) for | Field | Type | Required | Default | Description | | --- | --- | --- | --- | --- | -| `backend` | [StoreSpec](#storespec) | Yes | — | The underlying store wrap around. All content will first flow through self before forwarding to backend. In the event there is an error detected in self, the connection to the backend will be terminated, and early termination should always cause updates to fail on the backend. | +| `backend` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | The underlying store wrap around. All content will first flow through self before forwarding to backend. In the event there is an error detected in self, the connection to the backend will be terminated, and early termination should always cause updates to fail on the backend. | | `eviction_policy` | [EvictionPolicy](#evictionpolicy) | — | — | Policy used to evict items out of the store. Failure to set this value will cause items to never be removed from the store causing infinite memory usage. | ## FastSlowSpec | Field | Type | Required | Default | Description | | --- | --- | --- | --- | --- | -| `fast` | [StoreSpec](#storespec) | Yes | — | Fast store that will be attempted to be contacted before reaching out to the `slow` store. | +| `fast` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | Fast store that will be attempted to be contacted before reaching out to the `slow` store. | | `fast_direction` | [StoreDirection](#storedirection) | — | `"both"` | How to handle the fast store. This can be useful to set to Get for worker nodes such that results are persisted to the slow store only. | -| `slow` | [StoreSpec](#storespec) | Yes | — | If the object does not exist in the `fast` store it will try to get it from this store. | +| `slow` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | If the object does not exist in the `fast` store it will try to get it from this store. | | `slow_direction` | [StoreDirection](#storedirection) | — | `"both"` | How to handle the slow store. This can be useful if creating a diode and you wish to have an upstream read only store. | ## ShardSpec @@ -781,8 +781,8 @@ See [`experimental_cloud_object_store`](#experimental_cloud_object_store-1) for | Field | Type | Required | Default | Description | | --- | --- | --- | --- | --- | | `size` | integer (uint64) | Yes | — | Size to partition the data on. | -| `lower_store` | [StoreSpec](#storespec) | Yes | — | Store to send data when object is < (less than) size. | -| `upper_store` | [StoreSpec](#storespec) | Yes | — | Store to send data when object is >= (less than eq) size. | +| `lower_store` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | Store to send data when object is < (less than) size. | +| `upper_store` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | Store to send data when object is >= (less than eq) size. | ## GrpcSpec @@ -955,587 +955,6 @@ Listener for HTTP/HTTPS/HTTP2 sockets. | `"sha256"` | Use the sha256 hash function. [https://en.wikipedia.org/wiki/SHA-2](https://en.wikipedia.org/wiki/SHA-2) | | `"blake3"` | Use the blake3 hash function. [https://en.wikipedia.org/wiki/BLAKE_(hash_function)](https://en.wikipedia.org/wiki/BLAKE_(hash_function)) | -## StoreSpec - -Plus exactly one of the following variants (the key selects the variant): - -### `cache_metrics` - -Cache metrics store wraps another store and emits low-cardinality -OpenTelemetry cache operation metrics for the wrapped store. - -This wrapper is opt-in. Stores that are not explicitly wrapped by -`cache_metrics` are constructed exactly as they are without this -wrapper and do not pay its hot-path timing or recording cost. - -**Example JSON5 config:** -```json5 -"cache_metrics": { - "cache_type": "cas", - "backend": { - "filesystem": { - "content_path": "~/.cache/nativelink/content_path-cas", - "temp_path": "~/.cache/nativelink/tmp_path-cas" - } - } -} -``` - -**Type:** [CacheMetricsSpec](#cachemetricsspec) - -### `memory` - -Memory store will store all data in a hash map in memory. - -**Example JSON5 config:** -```json5 -"memory": { - "eviction_policy": { - "max_bytes": "10mb", - } -} -``` - -**Type:** [MemorySpec](#memoryspec) - -### `experimental_cloud_object_store` - -A generic blob store that will store files on the cloud -provider. This configuration will never delete files, so you are -responsible for purging old files in other ways. -It supports the following backends: - -1. **Amazon S3:** - S3 store will use Amazon's S3 service as a backend to store - the files. This configuration can be used to share files - across multiple instances. Uses system certificates for TLS - verification via `rustls-platform-verifier`. - - **Example JSON5 config:** - ```json5 - "experimental_cloud_object_store": { - "provider": "aws", - "region": "eu-north-1", - "bucket": "crossplane-bucket-af79aeca9", - "key_prefix": "test-prefix-index/", - "retry": { - "max_retries": 6, - "delay": 0.3, - "jitter": 0.5 - }, - "multipart_max_concurrent_uploads": 10 - } - ``` - -2. **Google Cloud Storage:** - GCS store uses Google's GCS service as a backend to store - the files. This configuration can be used to share files - across multiple instances. - - **Example JSON5 config:** - ```json5 - "experimental_cloud_object_store": { - "provider": "gcs", - "bucket": "test-bucket", - "key_prefix": "test-prefix-index/", - "retry": { - "max_retries": 6, - "delay": 0.3, - "jitter": 0.5 - }, - "multipart_max_concurrent_uploads": 10 - } - ``` - -3. **Azure Blob Store:** - Azure Blob store will use Microsoft's Azure Blob service as a - backend to store the files. This configuration can be used to - share files across multiple instances. - - **Example JSON5 config:** - ```json5 - "experimental_cloud_object_store": { - "provider": "azure", - "account_name": "cloudshell1393657559", - "container": "simple-test-container", - "key_prefix": "folder/", - "retry": { - "max_retries": 6, - "delay": 0.3, - "jitter": 0.5 - }, - "multipart_max_concurrent_uploads": 10 - } - ``` - -4. **`NetApp` ONTAP S3** - `NetApp` ONTAP S3 store will use ONTAP's S3-compatible storage as a backend - to store files. This store is specifically configured for ONTAP's S3 requirements - including custom TLS configuration, credentials management, and proper vserver - configuration. - - This store uses AWS environment variables for credentials: - - `AWS_ACCESS_KEY_ID` - - `AWS_SECRET_ACCESS_KEY` - - `AWS_DEFAULT_REGION` - - **Example JSON5 config:** - ```json5 - "experimental_cloud_object_store": { - "provider": "ontap", - "endpoint": "https://ontap-s3-endpoint:443", - "vserver_name": "your-vserver", - "bucket": "your-bucket", - "root_certificates": "/path/to/certs.pem", // Optional - "key_prefix": "test-prefix/", // Optional - "retry": { - "max_retries": 6, - "delay": 0.3, - "jitter": 0.5 - }, - "multipart_max_concurrent_uploads": 10 - } - ``` - -**Type:** [ExperimentalCloudObjectSpec](#experimentalcloudobjectspec) - -### `ontap_s3_existence_cache` - -ONTAP S3 Existence Cache provides a caching layer on top of the ONTAP S3 store -to optimize repeated existence checks. It maintains an in-memory cache of object -digests and periodically syncs this cache to disk for persistence. - -The cache helps reduce latency for repeated calls to check object existence, -while still ensuring eventual consistency with the underlying ONTAP S3 store. - -Example JSON5 config: -```json5 -"ontap_s3_existence_cache": { - "index_path": "/path/to/cache/index.json", - "sync_interval_seconds": 300, - "backend": { - "endpoint": "https://ontap-s3-endpoint:443", - "vserver_name": "your-vserver", - "bucket": "your-bucket", - "key_prefix": "test-prefix/" - } -} -``` - -**Type:** [OntapS3ExistenceCacheSpec](#ontaps3existencecachespec) - -### `verify` - -Verify store is used to apply verifications to an underlying -store implementation. It is strongly encouraged to validate -as much data as you can before accepting data from a client, -failing to do so may cause the data in the store to be -populated with invalid data causing all kinds of problems. - -The suggested configuration is to have the CAS validate the -hash and size and the AC validate nothing. - -**Example JSON5 config:** -```json5 -"verify": { - "backend": { - "memory": { - "eviction_policy": { - "max_bytes": "500mb" - } - }, - }, - "verify_size": true, - "verify_hash": true -} -``` - -**Type:** [VerifySpec](#verifyspec) - -### `completeness_checking` - -Completeness checking store verifies if the -output files & folders exist in the CAS before forwarding -the request to the underlying store. -Note: This store should only be used on AC stores. - -**Example JSON5 config:** -```json5 -"completeness_checking": { - "backend": { - "filesystem": { - "content_path": "~/.cache/nativelink/content_path-ac", - "temp_path": "~/.cache/nativelink/tmp_path-ac", - "eviction_policy": { - "max_bytes": "500mb", - } - } - }, - "cas_store": { - "ref_store": { - "name": "CAS_MAIN_STORE" - } - } -} -``` - -**Type:** [CompletenessCheckingSpec](#completenesscheckingspec) - -### `compression` - -A compression store that will compress the data inbound and -outbound. There will be a non-trivial cost to compress and -decompress the data, but in many cases if the final store is -a store that requires network transport and/or storage space -is a concern it is often faster and more efficient to use this -store before those stores. - -**Example JSON5 config:** -```json5 -"compression": { - "compression_algorithm": { - "lz4": {} - }, - "backend": { - "filesystem": { - "content_path": "/tmp/nativelink/data/content_path-cas", - "temp_path": "/tmp/nativelink/data/tmp_path-cas", - "eviction_policy": { - "max_bytes": "2gb", - } - } - } -} -``` - -**Type:** [CompressionSpec](#compressionspec) - -### `dedup` - -A dedup store will take the inputs and run a rolling hash -algorithm on them to slice the input into smaller parts then -run a sha256 algorithm on the slice and if the object doesn't -already exist, upload the slice to the `content_store` using -a new digest of just the slice. Once all parts exist, an -Action-Cache-like digest will be built and uploaded to the -`index_store` which will contain a reference to each -chunk/digest of the uploaded file. Downloading a request will -first grab the index from the `index_store`, and forward the -download content of each chunk as if it were one file. - -This store is exceptionally good when the following conditions -are met: -* Content is mostly the same (inserts, updates, deletes are ok) -* Content is not compressed or encrypted -* Uploading or downloading from `content_store` is the bottleneck. - -Note: This store pairs well when used with `CompressionSpec` as -the `content_store`, but never put `DedupSpec` as the backend of -`CompressionSpec` as it will negate all the gains. - -Note: When running `.has()` on this store, it will only check -to see if the entry exists in the `index_store` and not check -if the individual chunks exist in the `content_store`. - -**Example JSON5 config:** -```json5 -"dedup": { - "index_store": { - "memory": { - "eviction_policy": { - "max_bytes": "1GB", - } - } - }, - "content_store": { - "compression": { - "compression_algorithm": { - "lz4": {} - }, - "backend": { - "fast_slow": { - "fast": { - "memory": { - "eviction_policy": { - "max_bytes": "500MB", - } - } - }, - "slow": { - "filesystem": { - "content_path": "/tmp/nativelink/data/content_path-content", - "temp_path": "/tmp/nativelink/data/tmp_path-content", - "eviction_policy": { - "max_bytes": "2gb" - } - } - } - } - } - } - } -} -``` - -**Type:** [DedupSpec](#dedupspec) - -### `existence_cache` - -Existence store will wrap around another store and cache calls -to has so that subsequent `has_with_results` calls will be -faster. This is useful for cases when you have a store that -is slow to respond to has calls. -Note: This store should only be used on CAS stores. - -**Example JSON5 config:** -```json5 -"existence_cache": { - "backend": { - "memory": { - "eviction_policy": { - "max_bytes": "500mb", - } - } - }, - // Note this is the existence store policy, not the backend policy - "eviction_policy": { - "max_seconds": 100, - } -} -``` - -**Type:** [ExistenceCacheSpec](#existencecachespec) - -### `fast_slow` - -`FastSlow` store will first try to fetch the data from the `fast` -store and then if it does not exist try the `slow` store. -When the object does exist in the `slow` store, it will copy -the data to the `fast` store while returning the data. -This store should be thought of as a store that "buffers" -the data to the `fast` store. -On uploads it will mirror data to both `fast` and `slow` stores. - -WARNING: If you need data to always exist in the `slow` store -for something like remote execution, be careful because this -store will never check to see if the objects exist in the -`slow` store if it exists in the `fast` store (i.e., it assumes -that if an object exists in the `fast` store it will exist in -the `slow` store). - -***Example JSON5 config:*** -```json5 -"fast_slow": { - "fast": { - "filesystem": { - "content_path": "/tmp/nativelink/data/content_path-index", - "temp_path": "/tmp/nativelink/data/tmp_path-index", - "eviction_policy": { - "max_bytes": "500mb", - } - } - }, - "slow": { - "filesystem": { - "content_path": "/tmp/nativelink/data/content_path-index", - "temp_path": "/tmp/nativelink/data/tmp_path-index", - "eviction_policy": { - "max_bytes": "500mb", - } - } - } -} -``` - -**Type:** [FastSlowSpec](#fastslowspec) - -### `shard` - -Shards the data to multiple stores. This is useful for cases -when you want to distribute the load across multiple stores. -The digest hash is used to determine which store to send the -data to. - -**Example JSON5 config:** -```json5 -"shard": { - "stores": [ - { - "store": { - "memory": { - "eviction_policy": { - "max_bytes": "10mb" - }, - }, - }, - "weight": 1 - }] -} -``` - -**Type:** [ShardSpec](#shardspec) - -### `filesystem` - -Stores the data on the filesystem. This store is designed for -local persistent storage. Restarts of this program should restore -the previous state, meaning anything uploaded will be persistent -as long as the filesystem integrity holds. - -**Example JSON5 config:** -```json5 -"filesystem": { - "content_path": "/tmp/nativelink/data-worker-test/content_path-cas", - "temp_path": "/tmp/nativelink/data-worker-test/tmp_path-cas", - "eviction_policy": { - "max_bytes": "10gb", - } -} -``` - -**Type:** [FilesystemSpec](#filesystemspec) - -### `ref_store` - -Store used to reference a store in the root store manager. -This is useful for cases when you want to share a store in different -nested stores. Example, you may want to share the same memory store -used for the action cache, but use a `FastSlowSpec` and have the fast -store also share the memory store for efficiency. - -**Example JSON5 config:** -```json5 -"ref_store": { - "name": "FS_CONTENT_STORE" -} -``` - -**Type:** [RefSpec](#refspec) - -### `size_partitioning` - -Uses the size field of the digest to separate which store to send the -data. This is useful for cases when you'd like to put small objects -in one store and large objects in another store. This should only be -used if the size field is the real size of the content, in other -words, don't use on AC (Action Cache) stores. Any store where you can -safely use `VerifySpec.verify_size = true`, this store should be safe -to use (i.e., CAS stores). - -**Example JSON5 config:** -```json5 -"size_partitioning": { - "size": "128mib", - "lower_store": { - "memory": { - "eviction_policy": { - "max_bytes": "${NATIVELINK_CAS_MEMORY_CONTENT_LIMIT:-100mb}" - } - } - }, - "upper_store": { - /// This store discards data larger than 128mib. - "noop": {} - } -} -``` - -**Type:** [SizePartitioningSpec](#sizepartitioningspec) - -### `grpc` - -This store will pass-through calls to another GRPC store. This store -is not designed to be used as a sub-store of another store, but it -does satisfy the interface and will likely work. - -One major GOTCHA is that some stores use a special function on this -store to get the size of the underlying object, which is only reliable -when this store is serving the a CAS store, not an AC store. If using -this store directly without being a child of any store there are no -side effects and is the most efficient way to use it. - -**Example JSON5 config:** -```json5 -"grpc": { - "instance_name": "main", - "endpoints": [ - {"address": "grpc://${CAS_ENDPOINT:-127.0.0.1}:50051"} - ], - "connections_per_endpoint": "5", - "rpc_timeout_s": "5m", - "store_type": "ac", - // Static headers attached to every outgoing request to the upstream - // remote cache. Useful for fixed service-account credentials. - "headers": { - "authorization": "Bearer my-static-token" - }, - // Header names to copy from the inbound client request and forward to - // the upstream remote cache. Use this to pass through dynamic - // credentials such as a JWT sent by the build client. - "forward_headers": ["authorization", "x-custom-token"] -} -``` - -**Type:** [GrpcSpec](#grpcspec) - -### `redis_store` - -Stores data in any stores compatible with Redis APIs. - -Pairs well with `SizePartitioning` and/or `FastSlow` stores. -Ideal for accepting small object sizes as most Redis store -services have a max file upload of between 256Mb-512Mb. - -**Example JSON5 config:** -```json5 -"redis_store": { - "addresses": [ - "redis://127.0.0.1:6379/", - ], - "max_client_permits": 1000, -} -``` - -**Type:** [RedisSpec](#redisspec) - -### `noop` - -Noop store is a store that sends streams into the void and all data -retrieval will return 404 (`NotFound`). This can be useful for cases -where you may need to partition your data and part of your data needs -to be discarded. - -**Example JSON5 config:** -```json5 -"noop": {} -``` - -**Type:** [NoopSpec](#noopspec) - -### `experimental_mongo` - -Experimental `MongoDB` store implementation. - -This store uses `MongoDB` as a backend for storing data. It supports -both CAS (Content Addressable Storage) and scheduler data with -optional change streams for real-time updates. - -**Example JSON5 config:** -```json5 -"experimental_mongo": { - "connection_string": "mongodb://localhost:27017", - "database": "nativelink", - "cas_collection": "cas", - "key_prefix": "cas:", - "read_chunk_size": 65536, - "max_concurrent_uploads": 10, - "enable_change_streams": false, - "max_requests": "100" -} -``` - -**Type:** [ExperimentalMongoSpec](#experimentalmongospec) - ## EvictionPolicy Eviction policy always works on LRU (Least Recently Used). Any time an entry @@ -1630,7 +1049,7 @@ Configuration for an individual shard of the store. | Field | Type | Required | Default | Description | | --- | --- | --- | --- | --- | -| `store` | [StoreSpec](#storespec) | Yes | — | Store to shard the data to. | +| `store` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | Store to shard the data to. | | `weight` | integer (uint32) | — | 1 | The weight of the store. This is used to determine how much data should be sent to the store. The actual percentage is the sum of all the store's weights divided by the individual store's weight. | ## GrpcEndpoint diff --git a/web/apps/docs/content/docs/reference/nativelink-config/v1.4.0.mdx b/web/apps/docs/content/docs/reference/nativelink-config/v1.4.0.mdx index 789716942..a5f3844f6 100644 --- a/web/apps/docs/content/docs/reference/nativelink-config/v1.4.0.mdx +++ b/web/apps/docs/content/docs/reference/nativelink-config/v1.4.0.mdx @@ -151,8 +151,8 @@ It supports the following backends: } ``` -4. **`NetApp` ONTAP S3** - `NetApp` ONTAP S3 store will use ONTAP's S3-compatible storage as a backend +4. **NetApp ONTAP S3:** + NetApp ONTAP S3 store will use ONTAP's S3-compatible storage as a backend to store files. This store is specifically configured for ONTAP's S3 requirements including custom TLS configuration, credentials management, and proper vserver configuration. @@ -688,7 +688,7 @@ Plus exactly one of the following variants (the key selects the variant): | Field | Type | Required | Default | Description | | --- | --- | --- | --- | --- | | `cache_type` | string | Yes | — | Low-cardinality cache type label for metrics, for example `cas` or `ac`. | -| `backend` | [StoreSpec](#storespec) | Yes | — | Store to wrap with cache operation metrics. | +| `backend` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | Store to wrap with cache operation metrics. | ## MemorySpec @@ -712,7 +712,7 @@ See [`experimental_cloud_object_store`](#experimental_cloud_object_store-1) for | Field | Type | Required | Default | Description | | --- | --- | --- | --- | --- | -| `backend` | [StoreSpec](#storespec) | Yes | — | The underlying store wrap around. All content will first flow through self before forwarding to backend. In the event there is an error detected in self, the connection to the backend will be terminated, and early termination should always cause updates to fail on the backend. | +| `backend` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | The underlying store wrap around. All content will first flow through self before forwarding to backend. In the event there is an error detected in self, the connection to the backend will be terminated, and early termination should always cause updates to fail on the backend. | | `verify_size` | boolean | — | `false` | If set the store will verify the size of the data before accepting an upload of data. | | `verify_hash` | boolean | — | `false` | If the data should be hashed and verify that the key matches the computed hash. The hash function is automatically determined based request and if not set will use the global default. | @@ -720,22 +720,22 @@ See [`experimental_cloud_object_store`](#experimental_cloud_object_store-1) for | Field | Type | Required | Default | Description | | --- | --- | --- | --- | --- | -| `backend` | [StoreSpec](#storespec) | Yes | — | The underlying store that will have it's results validated before sending to client. | -| `cas_store` | [StoreSpec](#storespec) | Yes | — | When a request is made, the results are decoded and all output digests/files are verified to exist in this CAS store before returning success. | +| `backend` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | The underlying store that will have it's results validated before sending to client. | +| `cas_store` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | When a request is made, the results are decoded and all output digests/files are verified to exist in this CAS store before returning success. | ## CompressionSpec | Field | Type | Required | Default | Description | | --- | --- | --- | --- | --- | -| `backend` | [StoreSpec](#storespec) | Yes | — | The underlying store wrap around. All content will first flow through self before forwarding to backend. In the event there is an error detected in self, the connection to the backend will be terminated, and early termination should always cause updates to fail on the backend. | +| `backend` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | The underlying store wrap around. All content will first flow through self before forwarding to backend. In the event there is an error detected in self, the connection to the backend will be terminated, and early termination should always cause updates to fail on the backend. | | `compression_algorithm` | [CompressionAlgorithm](#compressionalgorithm) | Yes | — | The compression algorithm to use. | ## DedupSpec | Field | Type | Required | Default | Description | | --- | --- | --- | --- | --- | -| `index_store` | [StoreSpec](#storespec) | Yes | — | Store used to store the index of each dedup slice. This store should generally be fast and small. | -| `content_store` | [StoreSpec](#storespec) | Yes | — | The store where the individual chunks will be uploaded. This store should generally be the slower & larger store. | +| `index_store` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | Store used to store the index of each dedup slice. This store should generally be fast and small. | +| `content_store` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | The store where the individual chunks will be uploaded. This store should generally be the slower & larger store. | | `min_size` | integer (uint32) | — | 65536 (64k) | Minimum size that a chunk will be when slicing up the content. Note: This setting can be increased to improve performance because it will actually not check this number of bytes when deciding where to partition the data. | | `normal_size` | integer (uint32) | — | 262144 (256k) | A best-effort attempt will be made to keep the average size of the chunks to this number. It is not a guarantee, but a slight attempt will be made. | | `max_size` | integer (uint32) | — | 524288 (512k) | Maximum size a chunk is allowed to be. | @@ -745,16 +745,16 @@ See [`experimental_cloud_object_store`](#experimental_cloud_object_store-1) for | Field | Type | Required | Default | Description | | --- | --- | --- | --- | --- | -| `backend` | [StoreSpec](#storespec) | Yes | — | The underlying store wrap around. All content will first flow through self before forwarding to backend. In the event there is an error detected in self, the connection to the backend will be terminated, and early termination should always cause updates to fail on the backend. | +| `backend` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | The underlying store wrap around. All content will first flow through self before forwarding to backend. In the event there is an error detected in self, the connection to the backend will be terminated, and early termination should always cause updates to fail on the backend. | | `eviction_policy` | [EvictionPolicy](#evictionpolicy) | — | — | Policy used to evict items out of the store. Failure to set this value will cause items to never be removed from the store causing infinite memory usage. | ## FastSlowSpec | Field | Type | Required | Default | Description | | --- | --- | --- | --- | --- | -| `fast` | [StoreSpec](#storespec) | Yes | — | Fast store that will be attempted to be contacted before reaching out to the `slow` store. | +| `fast` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | Fast store that will be attempted to be contacted before reaching out to the `slow` store. | | `fast_direction` | [StoreDirection](#storedirection) | — | `"both"` | How to handle the fast store. This can be useful to set to Get for worker nodes such that results are persisted to the slow store only. | -| `slow` | [StoreSpec](#storespec) | Yes | — | If the object does not exist in the `fast` store it will try to get it from this store. | +| `slow` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | If the object does not exist in the `fast` store it will try to get it from this store. | | `slow_direction` | [StoreDirection](#storedirection) | — | `"both"` | How to handle the slow store. This can be useful if creating a diode and you wish to have an upstream read only store. | ## ShardSpec @@ -785,8 +785,8 @@ See [`experimental_cloud_object_store`](#experimental_cloud_object_store-1) for | Field | Type | Required | Default | Description | | --- | --- | --- | --- | --- | | `size` | integer (uint64) | Yes | — | Size to partition the data on. | -| `lower_store` | [StoreSpec](#storespec) | Yes | — | Store to send data when object is < (less than) size. | -| `upper_store` | [StoreSpec](#storespec) | Yes | — | Store to send data when object is >= (less than eq) size. | +| `lower_store` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | Store to send data when object is < (less than) size. | +| `upper_store` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | Store to send data when object is >= (less than eq) size. | ## GrpcSpec @@ -969,587 +969,6 @@ Listener for HTTP/HTTPS/HTTP2 sockets. | `"sha256"` | Use the sha256 hash function. [https://en.wikipedia.org/wiki/SHA-2](https://en.wikipedia.org/wiki/SHA-2) | | `"blake3"` | Use the blake3 hash function. [https://en.wikipedia.org/wiki/BLAKE_(hash_function)](https://en.wikipedia.org/wiki/BLAKE_(hash_function)) | -## StoreSpec - -Plus exactly one of the following variants (the key selects the variant): - -### `cache_metrics` - -Cache metrics store wraps another store and emits low-cardinality -OpenTelemetry cache operation metrics for the wrapped store. - -This wrapper is opt-in. Stores that are not explicitly wrapped by -`cache_metrics` are constructed exactly as they are without this -wrapper and do not pay its hot-path timing or recording cost. - -**Example JSON5 config:** -```json5 -"cache_metrics": { - "cache_type": "cas", - "backend": { - "filesystem": { - "content_path": "~/.cache/nativelink/content_path-cas", - "temp_path": "~/.cache/nativelink/tmp_path-cas" - } - } -} -``` - -**Type:** [CacheMetricsSpec](#cachemetricsspec) - -### `memory` - -Memory store will store all data in a hash map in memory. - -**Example JSON5 config:** -```json5 -"memory": { - "eviction_policy": { - "max_bytes": "10mb", - } -} -``` - -**Type:** [MemorySpec](#memoryspec) - -### `experimental_cloud_object_store` - -A generic blob store that will store files on the cloud -provider. This configuration will never delete files, so you are -responsible for purging old files in other ways. -It supports the following backends: - -1. **Amazon S3:** - S3 store will use Amazon's S3 service as a backend to store - the files. This configuration can be used to share files - across multiple instances. Uses system certificates for TLS - verification via `rustls-platform-verifier`. - - **Example JSON5 config:** - ```json5 - "experimental_cloud_object_store": { - "provider": "aws", - "region": "eu-north-1", - "bucket": "crossplane-bucket-af79aeca9", - "key_prefix": "test-prefix-index/", - "retry": { - "max_retries": 6, - "delay": 0.3, - "jitter": 0.5 - }, - "multipart_max_concurrent_uploads": 10 - } - ``` - -2. **Google Cloud Storage:** - GCS store uses Google's GCS service as a backend to store - the files. This configuration can be used to share files - across multiple instances. - - **Example JSON5 config:** - ```json5 - "experimental_cloud_object_store": { - "provider": "gcs", - "bucket": "test-bucket", - "key_prefix": "test-prefix-index/", - "retry": { - "max_retries": 6, - "delay": 0.3, - "jitter": 0.5 - }, - "multipart_max_concurrent_uploads": 10 - } - ``` - -3. **Azure Blob Store:** - Azure Blob store will use Microsoft's Azure Blob service as a - backend to store the files. This configuration can be used to - share files across multiple instances. - - **Example JSON5 config:** - ```json5 - "experimental_cloud_object_store": { - "provider": "azure", - "account_name": "cloudshell1393657559", - "container": "simple-test-container", - "key_prefix": "folder/", - "retry": { - "max_retries": 6, - "delay": 0.3, - "jitter": 0.5 - }, - "multipart_max_concurrent_uploads": 10 - } - ``` - -4. **`NetApp` ONTAP S3** - `NetApp` ONTAP S3 store will use ONTAP's S3-compatible storage as a backend - to store files. This store is specifically configured for ONTAP's S3 requirements - including custom TLS configuration, credentials management, and proper vserver - configuration. - - This store uses AWS environment variables for credentials: - - `AWS_ACCESS_KEY_ID` - - `AWS_SECRET_ACCESS_KEY` - - `AWS_DEFAULT_REGION` - - **Example JSON5 config:** - ```json5 - "experimental_cloud_object_store": { - "provider": "ontap", - "endpoint": "https://ontap-s3-endpoint:443", - "vserver_name": "your-vserver", - "bucket": "your-bucket", - "root_certificates": "/path/to/certs.pem", // Optional - "key_prefix": "test-prefix/", // Optional - "retry": { - "max_retries": 6, - "delay": 0.3, - "jitter": 0.5 - }, - "multipart_max_concurrent_uploads": 10 - } - ``` - -**Type:** [ExperimentalCloudObjectSpec](#experimentalcloudobjectspec) - -### `ontap_s3_existence_cache` - -ONTAP S3 Existence Cache provides a caching layer on top of the ONTAP S3 store -to optimize repeated existence checks. It maintains an in-memory cache of object -digests and periodically syncs this cache to disk for persistence. - -The cache helps reduce latency for repeated calls to check object existence, -while still ensuring eventual consistency with the underlying ONTAP S3 store. - -Example JSON5 config: -```json5 -"ontap_s3_existence_cache": { - "index_path": "/path/to/cache/index.json", - "sync_interval_seconds": 300, - "backend": { - "endpoint": "https://ontap-s3-endpoint:443", - "vserver_name": "your-vserver", - "bucket": "your-bucket", - "key_prefix": "test-prefix/" - } -} -``` - -**Type:** [OntapS3ExistenceCacheSpec](#ontaps3existencecachespec) - -### `verify` - -Verify store is used to apply verifications to an underlying -store implementation. It is strongly encouraged to validate -as much data as you can before accepting data from a client, -failing to do so may cause the data in the store to be -populated with invalid data causing all kinds of problems. - -The suggested configuration is to have the CAS validate the -hash and size and the AC validate nothing. - -**Example JSON5 config:** -```json5 -"verify": { - "backend": { - "memory": { - "eviction_policy": { - "max_bytes": "500mb" - } - }, - }, - "verify_size": true, - "verify_hash": true -} -``` - -**Type:** [VerifySpec](#verifyspec) - -### `completeness_checking` - -Completeness checking store verifies if the -output files & folders exist in the CAS before forwarding -the request to the underlying store. -Note: This store should only be used on AC stores. - -**Example JSON5 config:** -```json5 -"completeness_checking": { - "backend": { - "filesystem": { - "content_path": "~/.cache/nativelink/content_path-ac", - "temp_path": "~/.cache/nativelink/tmp_path-ac", - "eviction_policy": { - "max_bytes": "500mb", - } - } - }, - "cas_store": { - "ref_store": { - "name": "CAS_MAIN_STORE" - } - } -} -``` - -**Type:** [CompletenessCheckingSpec](#completenesscheckingspec) - -### `compression` - -A compression store that will compress the data inbound and -outbound. There will be a non-trivial cost to compress and -decompress the data, but in many cases if the final store is -a store that requires network transport and/or storage space -is a concern it is often faster and more efficient to use this -store before those stores. - -**Example JSON5 config:** -```json5 -"compression": { - "compression_algorithm": { - "lz4": {} - }, - "backend": { - "filesystem": { - "content_path": "/tmp/nativelink/data/content_path-cas", - "temp_path": "/tmp/nativelink/data/tmp_path-cas", - "eviction_policy": { - "max_bytes": "2gb", - } - } - } -} -``` - -**Type:** [CompressionSpec](#compressionspec) - -### `dedup` - -A dedup store will take the inputs and run a rolling hash -algorithm on them to slice the input into smaller parts then -run a sha256 algorithm on the slice and if the object doesn't -already exist, upload the slice to the `content_store` using -a new digest of just the slice. Once all parts exist, an -Action-Cache-like digest will be built and uploaded to the -`index_store` which will contain a reference to each -chunk/digest of the uploaded file. Downloading a request will -first grab the index from the `index_store`, and forward the -download content of each chunk as if it were one file. - -This store is exceptionally good when the following conditions -are met: -* Content is mostly the same (inserts, updates, deletes are ok) -* Content is not compressed or encrypted -* Uploading or downloading from `content_store` is the bottleneck. - -Note: This store pairs well when used with `CompressionSpec` as -the `content_store`, but never put `DedupSpec` as the backend of -`CompressionSpec` as it will negate all the gains. - -Note: When running `.has()` on this store, it will only check -to see if the entry exists in the `index_store` and not check -if the individual chunks exist in the `content_store`. - -**Example JSON5 config:** -```json5 -"dedup": { - "index_store": { - "memory": { - "eviction_policy": { - "max_bytes": "1GB", - } - } - }, - "content_store": { - "compression": { - "compression_algorithm": { - "lz4": {} - }, - "backend": { - "fast_slow": { - "fast": { - "memory": { - "eviction_policy": { - "max_bytes": "500MB", - } - } - }, - "slow": { - "filesystem": { - "content_path": "/tmp/nativelink/data/content_path-content", - "temp_path": "/tmp/nativelink/data/tmp_path-content", - "eviction_policy": { - "max_bytes": "2gb" - } - } - } - } - } - } - } -} -``` - -**Type:** [DedupSpec](#dedupspec) - -### `existence_cache` - -Existence store will wrap around another store and cache calls -to has so that subsequent `has_with_results` calls will be -faster. This is useful for cases when you have a store that -is slow to respond to has calls. -Note: This store should only be used on CAS stores. - -**Example JSON5 config:** -```json5 -"existence_cache": { - "backend": { - "memory": { - "eviction_policy": { - "max_bytes": "500mb", - } - } - }, - // Note this is the existence store policy, not the backend policy - "eviction_policy": { - "max_seconds": 100, - } -} -``` - -**Type:** [ExistenceCacheSpec](#existencecachespec) - -### `fast_slow` - -`FastSlow` store will first try to fetch the data from the `fast` -store and then if it does not exist try the `slow` store. -When the object does exist in the `slow` store, it will copy -the data to the `fast` store while returning the data. -This store should be thought of as a store that "buffers" -the data to the `fast` store. -On uploads it will mirror data to both `fast` and `slow` stores. - -WARNING: If you need data to always exist in the `slow` store -for something like remote execution, be careful because this -store will never check to see if the objects exist in the -`slow` store if it exists in the `fast` store (i.e., it assumes -that if an object exists in the `fast` store it will exist in -the `slow` store). - -***Example JSON5 config:*** -```json5 -"fast_slow": { - "fast": { - "filesystem": { - "content_path": "/tmp/nativelink/data/content_path-index", - "temp_path": "/tmp/nativelink/data/tmp_path-index", - "eviction_policy": { - "max_bytes": "500mb", - } - } - }, - "slow": { - "filesystem": { - "content_path": "/tmp/nativelink/data/content_path-index", - "temp_path": "/tmp/nativelink/data/tmp_path-index", - "eviction_policy": { - "max_bytes": "500mb", - } - } - } -} -``` - -**Type:** [FastSlowSpec](#fastslowspec) - -### `shard` - -Shards the data to multiple stores. This is useful for cases -when you want to distribute the load across multiple stores. -The digest hash is used to determine which store to send the -data to. - -**Example JSON5 config:** -```json5 -"shard": { - "stores": [ - { - "store": { - "memory": { - "eviction_policy": { - "max_bytes": "10mb" - }, - }, - }, - "weight": 1 - }] -} -``` - -**Type:** [ShardSpec](#shardspec) - -### `filesystem` - -Stores the data on the filesystem. This store is designed for -local persistent storage. Restarts of this program should restore -the previous state, meaning anything uploaded will be persistent -as long as the filesystem integrity holds. - -**Example JSON5 config:** -```json5 -"filesystem": { - "content_path": "/tmp/nativelink/data-worker-test/content_path-cas", - "temp_path": "/tmp/nativelink/data-worker-test/tmp_path-cas", - "eviction_policy": { - "max_bytes": "10gb", - } -} -``` - -**Type:** [FilesystemSpec](#filesystemspec) - -### `ref_store` - -Store used to reference a store in the root store manager. -This is useful for cases when you want to share a store in different -nested stores. Example, you may want to share the same memory store -used for the action cache, but use a `FastSlowSpec` and have the fast -store also share the memory store for efficiency. - -**Example JSON5 config:** -```json5 -"ref_store": { - "name": "FS_CONTENT_STORE" -} -``` - -**Type:** [RefSpec](#refspec) - -### `size_partitioning` - -Uses the size field of the digest to separate which store to send the -data. This is useful for cases when you'd like to put small objects -in one store and large objects in another store. This should only be -used if the size field is the real size of the content, in other -words, don't use on AC (Action Cache) stores. Any store where you can -safely use `VerifySpec.verify_size = true`, this store should be safe -to use (i.e., CAS stores). - -**Example JSON5 config:** -```json5 -"size_partitioning": { - "size": "128mib", - "lower_store": { - "memory": { - "eviction_policy": { - "max_bytes": "${NATIVELINK_CAS_MEMORY_CONTENT_LIMIT:-100mb}" - } - } - }, - "upper_store": { - /// This store discards data larger than 128mib. - "noop": {} - } -} -``` - -**Type:** [SizePartitioningSpec](#sizepartitioningspec) - -### `grpc` - -This store will pass-through calls to another GRPC store. This store -is not designed to be used as a sub-store of another store, but it -does satisfy the interface and will likely work. - -One major GOTCHA is that some stores use a special function on this -store to get the size of the underlying object, which is only reliable -when this store is serving the a CAS store, not an AC store. If using -this store directly without being a child of any store there are no -side effects and is the most efficient way to use it. - -**Example JSON5 config:** -```json5 -"grpc": { - "instance_name": "main", - "endpoints": [ - {"address": "grpc://${CAS_ENDPOINT:-127.0.0.1}:50051"} - ], - "connections_per_endpoint": "5", - "rpc_timeout_s": "5m", - "store_type": "ac", - // Static headers attached to every outgoing request to the upstream - // remote cache. Useful for fixed service-account credentials. - "headers": { - "authorization": "Bearer my-static-token" - }, - // Header names to copy from the inbound client request and forward to - // the upstream remote cache. Use this to pass through dynamic - // credentials such as a JWT sent by the build client. - "forward_headers": ["authorization", "x-custom-token"] -} -``` - -**Type:** [GrpcSpec](#grpcspec) - -### `redis_store` - -Stores data in any stores compatible with Redis APIs. - -Pairs well with `SizePartitioning` and/or `FastSlow` stores. -Ideal for accepting small object sizes as most Redis store -services have a max file upload of between 256Mb-512Mb. - -**Example JSON5 config:** -```json5 -"redis_store": { - "addresses": [ - "redis://127.0.0.1:6379/", - ], - "max_client_permits": 1000, -} -``` - -**Type:** [RedisSpec](#redisspec) - -### `noop` - -Noop store is a store that sends streams into the void and all data -retrieval will return 404 (`NotFound`). This can be useful for cases -where you may need to partition your data and part of your data needs -to be discarded. - -**Example JSON5 config:** -```json5 -"noop": {} -``` - -**Type:** [NoopSpec](#noopspec) - -### `experimental_mongo` - -Experimental `MongoDB` store implementation. - -This store uses `MongoDB` as a backend for storing data. It supports -both CAS (Content Addressable Storage) and scheduler data with -optional change streams for real-time updates. - -**Example JSON5 config:** -```json5 -"experimental_mongo": { - "connection_string": "mongodb://localhost:27017", - "database": "nativelink", - "cas_collection": "cas", - "key_prefix": "cas:", - "read_chunk_size": 65536, - "max_concurrent_uploads": 10, - "enable_change_streams": false, - "max_requests": "100" -} -``` - -**Type:** [ExperimentalMongoSpec](#experimentalmongospec) - ## EvictionPolicy Eviction policy always works on LRU (Least Recently Used). Any time an entry @@ -1644,7 +1063,7 @@ Configuration for an individual shard of the store. | Field | Type | Required | Default | Description | | --- | --- | --- | --- | --- | -| `store` | [StoreSpec](#storespec) | Yes | — | Store to shard the data to. | +| `store` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | Store to shard the data to. | | `weight` | integer (uint32) | — | 1 | The weight of the store. This is used to determine how much data should be sent to the store. The actual percentage is the sum of all the store's weights divided by the individual store's weight. | ## GrpcEndpoint diff --git a/web/apps/docs/content/docs/reference/nativelink-config/v1.5.0.mdx b/web/apps/docs/content/docs/reference/nativelink-config/v1.5.0.mdx index f83656b30..49ceb05cd 100644 --- a/web/apps/docs/content/docs/reference/nativelink-config/v1.5.0.mdx +++ b/web/apps/docs/content/docs/reference/nativelink-config/v1.5.0.mdx @@ -151,8 +151,8 @@ It supports the following backends: } ``` -4. **`NetApp` ONTAP S3** - `NetApp` ONTAP S3 store will use ONTAP's S3-compatible storage as a backend +4. **NetApp ONTAP S3:** + NetApp ONTAP S3 store will use ONTAP's S3-compatible storage as a backend to store files. This store is specifically configured for ONTAP's S3 requirements including custom TLS configuration, credentials management, and proper vserver configuration. @@ -688,7 +688,7 @@ Plus exactly one of the following variants (the key selects the variant): | Field | Type | Required | Default | Description | | --- | --- | --- | --- | --- | | `cache_type` | string | Yes | — | Low-cardinality cache type label for metrics, for example `cas` or `ac`. | -| `backend` | [StoreSpec](#storespec) | Yes | — | Store to wrap with cache operation metrics. | +| `backend` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | Store to wrap with cache operation metrics. | ## MemorySpec @@ -712,7 +712,7 @@ See [`experimental_cloud_object_store`](#experimental_cloud_object_store-1) for | Field | Type | Required | Default | Description | | --- | --- | --- | --- | --- | -| `backend` | [StoreSpec](#storespec) | Yes | — | The underlying store wrap around. All content will first flow through self before forwarding to backend. In the event there is an error detected in self, the connection to the backend will be terminated, and early termination should always cause updates to fail on the backend. | +| `backend` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | The underlying store wrap around. All content will first flow through self before forwarding to backend. In the event there is an error detected in self, the connection to the backend will be terminated, and early termination should always cause updates to fail on the backend. | | `verify_size` | boolean | — | `false` | If set the store will verify the size of the data before accepting an upload of data. | | `verify_hash` | boolean | — | `false` | If the data should be hashed and verify that the key matches the computed hash. The hash function is automatically determined based request and if not set will use the global default. | @@ -720,22 +720,22 @@ See [`experimental_cloud_object_store`](#experimental_cloud_object_store-1) for | Field | Type | Required | Default | Description | | --- | --- | --- | --- | --- | -| `backend` | [StoreSpec](#storespec) | Yes | — | The underlying store that will have it's results validated before sending to client. | -| `cas_store` | [StoreSpec](#storespec) | Yes | — | When a request is made, the results are decoded and all output digests/files are verified to exist in this CAS store before returning success. | +| `backend` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | The underlying store that will have it's results validated before sending to client. | +| `cas_store` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | When a request is made, the results are decoded and all output digests/files are verified to exist in this CAS store before returning success. | ## CompressionSpec | Field | Type | Required | Default | Description | | --- | --- | --- | --- | --- | -| `backend` | [StoreSpec](#storespec) | Yes | — | The underlying store wrap around. All content will first flow through self before forwarding to backend. In the event there is an error detected in self, the connection to the backend will be terminated, and early termination should always cause updates to fail on the backend. | +| `backend` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | The underlying store wrap around. All content will first flow through self before forwarding to backend. In the event there is an error detected in self, the connection to the backend will be terminated, and early termination should always cause updates to fail on the backend. | | `compression_algorithm` | [CompressionAlgorithm](#compressionalgorithm) | Yes | — | The compression algorithm to use. | ## DedupSpec | Field | Type | Required | Default | Description | | --- | --- | --- | --- | --- | -| `index_store` | [StoreSpec](#storespec) | Yes | — | Store used to store the index of each dedup slice. This store should generally be fast and small. | -| `content_store` | [StoreSpec](#storespec) | Yes | — | The store where the individual chunks will be uploaded. This store should generally be the slower & larger store. | +| `index_store` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | Store used to store the index of each dedup slice. This store should generally be fast and small. | +| `content_store` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | The store where the individual chunks will be uploaded. This store should generally be the slower & larger store. | | `min_size` | integer (uint32) | — | 65536 (64k) | Minimum size that a chunk will be when slicing up the content. Note: This setting can be increased to improve performance because it will actually not check this number of bytes when deciding where to partition the data. | | `normal_size` | integer (uint32) | — | 262144 (256k) | A best-effort attempt will be made to keep the average size of the chunks to this number. It is not a guarantee, but a slight attempt will be made. | | `max_size` | integer (uint32) | — | 524288 (512k) | Maximum size a chunk is allowed to be. | @@ -745,16 +745,16 @@ See [`experimental_cloud_object_store`](#experimental_cloud_object_store-1) for | Field | Type | Required | Default | Description | | --- | --- | --- | --- | --- | -| `backend` | [StoreSpec](#storespec) | Yes | — | The underlying store wrap around. All content will first flow through self before forwarding to backend. In the event there is an error detected in self, the connection to the backend will be terminated, and early termination should always cause updates to fail on the backend. | +| `backend` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | The underlying store wrap around. All content will first flow through self before forwarding to backend. In the event there is an error detected in self, the connection to the backend will be terminated, and early termination should always cause updates to fail on the backend. | | `eviction_policy` | [EvictionPolicy](#evictionpolicy) | — | — | Policy used to evict items out of the store. Failure to set this value will cause items to never be removed from the store causing infinite memory usage. | ## FastSlowSpec | Field | Type | Required | Default | Description | | --- | --- | --- | --- | --- | -| `fast` | [StoreSpec](#storespec) | Yes | — | Fast store that will be attempted to be contacted before reaching out to the `slow` store. | +| `fast` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | Fast store that will be attempted to be contacted before reaching out to the `slow` store. | | `fast_direction` | [StoreDirection](#storedirection) | — | `"both"` | How to handle the fast store. This can be useful to set to Get for worker nodes such that results are persisted to the slow store only. | -| `slow` | [StoreSpec](#storespec) | Yes | — | If the object does not exist in the `fast` store it will try to get it from this store. | +| `slow` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | If the object does not exist in the `fast` store it will try to get it from this store. | | `slow_direction` | [StoreDirection](#storedirection) | — | `"both"` | How to handle the slow store. This can be useful if creating a diode and you wish to have an upstream read only store. | | `bypass_dedup_threshold_bytes` | integer (uint64) | — | `0` | Reads of blobs at or above this size skip the leader/follower dedup map and stream straight from the slow store without populating the fast tier. `0` (the default) disables the bypass: every read goes through dedup, matching the prior behaviour. Enable it by setting a threshold — 256 MiB is a reasonable starting point for backends where large-blob dedup is a net loss (followers tend to time out anyway), but the right value is workload-dependent. | @@ -786,8 +786,8 @@ See [`experimental_cloud_object_store`](#experimental_cloud_object_store-1) for | Field | Type | Required | Default | Description | | --- | --- | --- | --- | --- | | `size` | integer (uint64) | Yes | — | Size to partition the data on. | -| `lower_store` | [StoreSpec](#storespec) | Yes | — | Store to send data when object is < (less than) size. | -| `upper_store` | [StoreSpec](#storespec) | Yes | — | Store to send data when object is >= (less than eq) size. | +| `lower_store` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | Store to send data when object is < (less than) size. | +| `upper_store` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | Store to send data when object is >= (less than eq) size. | ## GrpcSpec @@ -970,587 +970,6 @@ Listener for HTTP/HTTPS/HTTP2 sockets. | `"sha256"` | Use the sha256 hash function. [https://en.wikipedia.org/wiki/SHA-2](https://en.wikipedia.org/wiki/SHA-2) | | `"blake3"` | Use the blake3 hash function. [https://en.wikipedia.org/wiki/BLAKE_(hash_function)](https://en.wikipedia.org/wiki/BLAKE_(hash_function)) | -## StoreSpec - -Plus exactly one of the following variants (the key selects the variant): - -### `cache_metrics` - -Cache metrics store wraps another store and emits low-cardinality -OpenTelemetry cache operation metrics for the wrapped store. - -This wrapper is opt-in. Stores that are not explicitly wrapped by -`cache_metrics` are constructed exactly as they are without this -wrapper and do not pay its hot-path timing or recording cost. - -**Example JSON5 config:** -```json5 -"cache_metrics": { - "cache_type": "cas", - "backend": { - "filesystem": { - "content_path": "~/.cache/nativelink/content_path-cas", - "temp_path": "~/.cache/nativelink/tmp_path-cas" - } - } -} -``` - -**Type:** [CacheMetricsSpec](#cachemetricsspec) - -### `memory` - -Memory store will store all data in a hash map in memory. - -**Example JSON5 config:** -```json5 -"memory": { - "eviction_policy": { - "max_bytes": "10mb", - } -} -``` - -**Type:** [MemorySpec](#memoryspec) - -### `experimental_cloud_object_store` - -A generic blob store that will store files on the cloud -provider. This configuration will never delete files, so you are -responsible for purging old files in other ways. -It supports the following backends: - -1. **Amazon S3:** - S3 store will use Amazon's S3 service as a backend to store - the files. This configuration can be used to share files - across multiple instances. Uses system certificates for TLS - verification via `rustls-platform-verifier`. - - **Example JSON5 config:** - ```json5 - "experimental_cloud_object_store": { - "provider": "aws", - "region": "eu-north-1", - "bucket": "crossplane-bucket-af79aeca9", - "key_prefix": "test-prefix-index/", - "retry": { - "max_retries": 6, - "delay": 0.3, - "jitter": 0.5 - }, - "multipart_max_concurrent_uploads": 10 - } - ``` - -2. **Google Cloud Storage:** - GCS store uses Google's GCS service as a backend to store - the files. This configuration can be used to share files - across multiple instances. - - **Example JSON5 config:** - ```json5 - "experimental_cloud_object_store": { - "provider": "gcs", - "bucket": "test-bucket", - "key_prefix": "test-prefix-index/", - "retry": { - "max_retries": 6, - "delay": 0.3, - "jitter": 0.5 - }, - "multipart_max_concurrent_uploads": 10 - } - ``` - -3. **Azure Blob Store:** - Azure Blob store will use Microsoft's Azure Blob service as a - backend to store the files. This configuration can be used to - share files across multiple instances. - - **Example JSON5 config:** - ```json5 - "experimental_cloud_object_store": { - "provider": "azure", - "account_name": "cloudshell1393657559", - "container": "simple-test-container", - "key_prefix": "folder/", - "retry": { - "max_retries": 6, - "delay": 0.3, - "jitter": 0.5 - }, - "multipart_max_concurrent_uploads": 10 - } - ``` - -4. **`NetApp` ONTAP S3** - `NetApp` ONTAP S3 store will use ONTAP's S3-compatible storage as a backend - to store files. This store is specifically configured for ONTAP's S3 requirements - including custom TLS configuration, credentials management, and proper vserver - configuration. - - This store uses AWS environment variables for credentials: - - `AWS_ACCESS_KEY_ID` - - `AWS_SECRET_ACCESS_KEY` - - `AWS_DEFAULT_REGION` - - **Example JSON5 config:** - ```json5 - "experimental_cloud_object_store": { - "provider": "ontap", - "endpoint": "https://ontap-s3-endpoint:443", - "vserver_name": "your-vserver", - "bucket": "your-bucket", - "root_certificates": "/path/to/certs.pem", // Optional - "key_prefix": "test-prefix/", // Optional - "retry": { - "max_retries": 6, - "delay": 0.3, - "jitter": 0.5 - }, - "multipart_max_concurrent_uploads": 10 - } - ``` - -**Type:** [ExperimentalCloudObjectSpec](#experimentalcloudobjectspec) - -### `ontap_s3_existence_cache` - -ONTAP S3 Existence Cache provides a caching layer on top of the ONTAP S3 store -to optimize repeated existence checks. It maintains an in-memory cache of object -digests and periodically syncs this cache to disk for persistence. - -The cache helps reduce latency for repeated calls to check object existence, -while still ensuring eventual consistency with the underlying ONTAP S3 store. - -Example JSON5 config: -```json5 -"ontap_s3_existence_cache": { - "index_path": "/path/to/cache/index.json", - "sync_interval_seconds": 300, - "backend": { - "endpoint": "https://ontap-s3-endpoint:443", - "vserver_name": "your-vserver", - "bucket": "your-bucket", - "key_prefix": "test-prefix/" - } -} -``` - -**Type:** [OntapS3ExistenceCacheSpec](#ontaps3existencecachespec) - -### `verify` - -Verify store is used to apply verifications to an underlying -store implementation. It is strongly encouraged to validate -as much data as you can before accepting data from a client, -failing to do so may cause the data in the store to be -populated with invalid data causing all kinds of problems. - -The suggested configuration is to have the CAS validate the -hash and size and the AC validate nothing. - -**Example JSON5 config:** -```json5 -"verify": { - "backend": { - "memory": { - "eviction_policy": { - "max_bytes": "500mb" - } - }, - }, - "verify_size": true, - "verify_hash": true -} -``` - -**Type:** [VerifySpec](#verifyspec) - -### `completeness_checking` - -Completeness checking store verifies if the -output files & folders exist in the CAS before forwarding -the request to the underlying store. -Note: This store should only be used on AC stores. - -**Example JSON5 config:** -```json5 -"completeness_checking": { - "backend": { - "filesystem": { - "content_path": "~/.cache/nativelink/content_path-ac", - "temp_path": "~/.cache/nativelink/tmp_path-ac", - "eviction_policy": { - "max_bytes": "500mb", - } - } - }, - "cas_store": { - "ref_store": { - "name": "CAS_MAIN_STORE" - } - } -} -``` - -**Type:** [CompletenessCheckingSpec](#completenesscheckingspec) - -### `compression` - -A compression store that will compress the data inbound and -outbound. There will be a non-trivial cost to compress and -decompress the data, but in many cases if the final store is -a store that requires network transport and/or storage space -is a concern it is often faster and more efficient to use this -store before those stores. - -**Example JSON5 config:** -```json5 -"compression": { - "compression_algorithm": { - "lz4": {} - }, - "backend": { - "filesystem": { - "content_path": "/tmp/nativelink/data/content_path-cas", - "temp_path": "/tmp/nativelink/data/tmp_path-cas", - "eviction_policy": { - "max_bytes": "2gb", - } - } - } -} -``` - -**Type:** [CompressionSpec](#compressionspec) - -### `dedup` - -A dedup store will take the inputs and run a rolling hash -algorithm on them to slice the input into smaller parts then -run a sha256 algorithm on the slice and if the object doesn't -already exist, upload the slice to the `content_store` using -a new digest of just the slice. Once all parts exist, an -Action-Cache-like digest will be built and uploaded to the -`index_store` which will contain a reference to each -chunk/digest of the uploaded file. Downloading a request will -first grab the index from the `index_store`, and forward the -download content of each chunk as if it were one file. - -This store is exceptionally good when the following conditions -are met: -* Content is mostly the same (inserts, updates, deletes are ok) -* Content is not compressed or encrypted -* Uploading or downloading from `content_store` is the bottleneck. - -Note: This store pairs well when used with `CompressionSpec` as -the `content_store`, but never put `DedupSpec` as the backend of -`CompressionSpec` as it will negate all the gains. - -Note: When running `.has()` on this store, it will only check -to see if the entry exists in the `index_store` and not check -if the individual chunks exist in the `content_store`. - -**Example JSON5 config:** -```json5 -"dedup": { - "index_store": { - "memory": { - "eviction_policy": { - "max_bytes": "1GB", - } - } - }, - "content_store": { - "compression": { - "compression_algorithm": { - "lz4": {} - }, - "backend": { - "fast_slow": { - "fast": { - "memory": { - "eviction_policy": { - "max_bytes": "500MB", - } - } - }, - "slow": { - "filesystem": { - "content_path": "/tmp/nativelink/data/content_path-content", - "temp_path": "/tmp/nativelink/data/tmp_path-content", - "eviction_policy": { - "max_bytes": "2gb" - } - } - } - } - } - } - } -} -``` - -**Type:** [DedupSpec](#dedupspec) - -### `existence_cache` - -Existence store will wrap around another store and cache calls -to has so that subsequent `has_with_results` calls will be -faster. This is useful for cases when you have a store that -is slow to respond to has calls. -Note: This store should only be used on CAS stores. - -**Example JSON5 config:** -```json5 -"existence_cache": { - "backend": { - "memory": { - "eviction_policy": { - "max_bytes": "500mb", - } - } - }, - // Note this is the existence store policy, not the backend policy - "eviction_policy": { - "max_seconds": 100, - } -} -``` - -**Type:** [ExistenceCacheSpec](#existencecachespec) - -### `fast_slow` - -`FastSlow` store will first try to fetch the data from the `fast` -store and then if it does not exist try the `slow` store. -When the object does exist in the `slow` store, it will copy -the data to the `fast` store while returning the data. -This store should be thought of as a store that "buffers" -the data to the `fast` store. -On uploads it will mirror data to both `fast` and `slow` stores. - -WARNING: If you need data to always exist in the `slow` store -for something like remote execution, be careful because this -store will never check to see if the objects exist in the -`slow` store if it exists in the `fast` store (i.e., it assumes -that if an object exists in the `fast` store it will exist in -the `slow` store). - -***Example JSON5 config:*** -```json5 -"fast_slow": { - "fast": { - "filesystem": { - "content_path": "/tmp/nativelink/data/content_path-index", - "temp_path": "/tmp/nativelink/data/tmp_path-index", - "eviction_policy": { - "max_bytes": "500mb", - } - } - }, - "slow": { - "filesystem": { - "content_path": "/tmp/nativelink/data/content_path-index", - "temp_path": "/tmp/nativelink/data/tmp_path-index", - "eviction_policy": { - "max_bytes": "500mb", - } - } - } -} -``` - -**Type:** [FastSlowSpec](#fastslowspec) - -### `shard` - -Shards the data to multiple stores. This is useful for cases -when you want to distribute the load across multiple stores. -The digest hash is used to determine which store to send the -data to. - -**Example JSON5 config:** -```json5 -"shard": { - "stores": [ - { - "store": { - "memory": { - "eviction_policy": { - "max_bytes": "10mb" - }, - }, - }, - "weight": 1 - }] -} -``` - -**Type:** [ShardSpec](#shardspec) - -### `filesystem` - -Stores the data on the filesystem. This store is designed for -local persistent storage. Restarts of this program should restore -the previous state, meaning anything uploaded will be persistent -as long as the filesystem integrity holds. - -**Example JSON5 config:** -```json5 -"filesystem": { - "content_path": "/tmp/nativelink/data-worker-test/content_path-cas", - "temp_path": "/tmp/nativelink/data-worker-test/tmp_path-cas", - "eviction_policy": { - "max_bytes": "10gb", - } -} -``` - -**Type:** [FilesystemSpec](#filesystemspec) - -### `ref_store` - -Store used to reference a store in the root store manager. -This is useful for cases when you want to share a store in different -nested stores. Example, you may want to share the same memory store -used for the action cache, but use a `FastSlowSpec` and have the fast -store also share the memory store for efficiency. - -**Example JSON5 config:** -```json5 -"ref_store": { - "name": "FS_CONTENT_STORE" -} -``` - -**Type:** [RefSpec](#refspec) - -### `size_partitioning` - -Uses the size field of the digest to separate which store to send the -data. This is useful for cases when you'd like to put small objects -in one store and large objects in another store. This should only be -used if the size field is the real size of the content, in other -words, don't use on AC (Action Cache) stores. Any store where you can -safely use `VerifySpec.verify_size = true`, this store should be safe -to use (i.e., CAS stores). - -**Example JSON5 config:** -```json5 -"size_partitioning": { - "size": "128mib", - "lower_store": { - "memory": { - "eviction_policy": { - "max_bytes": "${NATIVELINK_CAS_MEMORY_CONTENT_LIMIT:-100mb}" - } - } - }, - "upper_store": { - /// This store discards data larger than 128mib. - "noop": {} - } -} -``` - -**Type:** [SizePartitioningSpec](#sizepartitioningspec) - -### `grpc` - -This store will pass-through calls to another GRPC store. This store -is not designed to be used as a sub-store of another store, but it -does satisfy the interface and will likely work. - -One major GOTCHA is that some stores use a special function on this -store to get the size of the underlying object, which is only reliable -when this store is serving the a CAS store, not an AC store. If using -this store directly without being a child of any store there are no -side effects and is the most efficient way to use it. - -**Example JSON5 config:** -```json5 -"grpc": { - "instance_name": "main", - "endpoints": [ - {"address": "grpc://${CAS_ENDPOINT:-127.0.0.1}:50051"} - ], - "connections_per_endpoint": "5", - "rpc_timeout_s": "5m", - "store_type": "ac", - // Static headers attached to every outgoing request to the upstream - // remote cache. Useful for fixed service-account credentials. - "headers": { - "authorization": "Bearer my-static-token" - }, - // Header names to copy from the inbound client request and forward to - // the upstream remote cache. Use this to pass through dynamic - // credentials such as a JWT sent by the build client. - "forward_headers": ["authorization", "x-custom-token"] -} -``` - -**Type:** [GrpcSpec](#grpcspec) - -### `redis_store` - -Stores data in any stores compatible with Redis APIs. - -Pairs well with `SizePartitioning` and/or `FastSlow` stores. -Ideal for accepting small object sizes as most Redis store -services have a max file upload of between 256Mb-512Mb. - -**Example JSON5 config:** -```json5 -"redis_store": { - "addresses": [ - "redis://127.0.0.1:6379/", - ], - "max_client_permits": 1000, -} -``` - -**Type:** [RedisSpec](#redisspec) - -### `noop` - -Noop store is a store that sends streams into the void and all data -retrieval will return 404 (`NotFound`). This can be useful for cases -where you may need to partition your data and part of your data needs -to be discarded. - -**Example JSON5 config:** -```json5 -"noop": {} -``` - -**Type:** [NoopSpec](#noopspec) - -### `experimental_mongo` - -Experimental `MongoDB` store implementation. - -This store uses `MongoDB` as a backend for storing data. It supports -both CAS (Content Addressable Storage) and scheduler data with -optional change streams for real-time updates. - -**Example JSON5 config:** -```json5 -"experimental_mongo": { - "connection_string": "mongodb://localhost:27017", - "database": "nativelink", - "cas_collection": "cas", - "key_prefix": "cas:", - "read_chunk_size": 65536, - "max_concurrent_uploads": 10, - "enable_change_streams": false, - "max_requests": "100" -} -``` - -**Type:** [ExperimentalMongoSpec](#experimentalmongospec) - ## EvictionPolicy Eviction policy always works on LRU (Least Recently Used). Any time an entry @@ -1645,7 +1064,7 @@ Configuration for an individual shard of the store. | Field | Type | Required | Default | Description | | --- | --- | --- | --- | --- | -| `store` | [StoreSpec](#storespec) | Yes | — | Store to shard the data to. | +| `store` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | Store to shard the data to. | | `weight` | integer (uint32) | — | 1 | The weight of the store. This is used to determine how much data should be sent to the store. The actual percentage is the sum of all the store's weights divided by the individual store's weight. | ## GrpcEndpoint diff --git a/web/apps/docs/content/docs/reference/nativelink-config/v1.5.1.mdx b/web/apps/docs/content/docs/reference/nativelink-config/v1.5.1.mdx index 998dc3d6e..c4e5d43d1 100644 --- a/web/apps/docs/content/docs/reference/nativelink-config/v1.5.1.mdx +++ b/web/apps/docs/content/docs/reference/nativelink-config/v1.5.1.mdx @@ -151,8 +151,8 @@ It supports the following backends: } ``` -4. **`NetApp` ONTAP S3** - `NetApp` ONTAP S3 store will use ONTAP's S3-compatible storage as a backend +4. **NetApp ONTAP S3:** + NetApp ONTAP S3 store will use ONTAP's S3-compatible storage as a backend to store files. This store is specifically configured for ONTAP's S3 requirements including custom TLS configuration, credentials management, and proper vserver configuration. @@ -688,7 +688,7 @@ Plus exactly one of the following variants (the key selects the variant): | Field | Type | Required | Default | Description | | --- | --- | --- | --- | --- | | `cache_type` | string | Yes | — | Low-cardinality cache type label for metrics, for example `cas` or `ac`. | -| `backend` | [StoreSpec](#storespec) | Yes | — | Store to wrap with cache operation metrics. | +| `backend` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | Store to wrap with cache operation metrics. | ## MemorySpec @@ -712,7 +712,7 @@ See [`experimental_cloud_object_store`](#experimental_cloud_object_store-1) for | Field | Type | Required | Default | Description | | --- | --- | --- | --- | --- | -| `backend` | [StoreSpec](#storespec) | Yes | — | The underlying store wrap around. All content will first flow through self before forwarding to backend. In the event there is an error detected in self, the connection to the backend will be terminated, and early termination should always cause updates to fail on the backend. | +| `backend` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | The underlying store wrap around. All content will first flow through self before forwarding to backend. In the event there is an error detected in self, the connection to the backend will be terminated, and early termination should always cause updates to fail on the backend. | | `verify_size` | boolean | — | `false` | If set the store will verify the size of the data before accepting an upload of data. | | `verify_hash` | boolean | — | `false` | If the data should be hashed and verify that the key matches the computed hash. The hash function is automatically determined based request and if not set will use the global default. | @@ -720,22 +720,22 @@ See [`experimental_cloud_object_store`](#experimental_cloud_object_store-1) for | Field | Type | Required | Default | Description | | --- | --- | --- | --- | --- | -| `backend` | [StoreSpec](#storespec) | Yes | — | The underlying store that will have it's results validated before sending to client. | -| `cas_store` | [StoreSpec](#storespec) | Yes | — | When a request is made, the results are decoded and all output digests/files are verified to exist in this CAS store before returning success. | +| `backend` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | The underlying store that will have it's results validated before sending to client. | +| `cas_store` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | When a request is made, the results are decoded and all output digests/files are verified to exist in this CAS store before returning success. | ## CompressionSpec | Field | Type | Required | Default | Description | | --- | --- | --- | --- | --- | -| `backend` | [StoreSpec](#storespec) | Yes | — | The underlying store wrap around. All content will first flow through self before forwarding to backend. In the event there is an error detected in self, the connection to the backend will be terminated, and early termination should always cause updates to fail on the backend. | +| `backend` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | The underlying store wrap around. All content will first flow through self before forwarding to backend. In the event there is an error detected in self, the connection to the backend will be terminated, and early termination should always cause updates to fail on the backend. | | `compression_algorithm` | [CompressionAlgorithm](#compressionalgorithm) | Yes | — | The compression algorithm to use. | ## DedupSpec | Field | Type | Required | Default | Description | | --- | --- | --- | --- | --- | -| `index_store` | [StoreSpec](#storespec) | Yes | — | Store used to store the index of each dedup slice. This store should generally be fast and small. | -| `content_store` | [StoreSpec](#storespec) | Yes | — | The store where the individual chunks will be uploaded. This store should generally be the slower & larger store. | +| `index_store` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | Store used to store the index of each dedup slice. This store should generally be fast and small. | +| `content_store` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | The store where the individual chunks will be uploaded. This store should generally be the slower & larger store. | | `min_size` | integer (uint32) | — | 65536 (64k) | Minimum size that a chunk will be when slicing up the content. Note: This setting can be increased to improve performance because it will actually not check this number of bytes when deciding where to partition the data. | | `normal_size` | integer (uint32) | — | 262144 (256k) | A best-effort attempt will be made to keep the average size of the chunks to this number. It is not a guarantee, but a slight attempt will be made. | | `max_size` | integer (uint32) | — | 524288 (512k) | Maximum size a chunk is allowed to be. | @@ -745,16 +745,16 @@ See [`experimental_cloud_object_store`](#experimental_cloud_object_store-1) for | Field | Type | Required | Default | Description | | --- | --- | --- | --- | --- | -| `backend` | [StoreSpec](#storespec) | Yes | — | The underlying store wrap around. All content will first flow through self before forwarding to backend. In the event there is an error detected in self, the connection to the backend will be terminated, and early termination should always cause updates to fail on the backend. | +| `backend` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | The underlying store wrap around. All content will first flow through self before forwarding to backend. In the event there is an error detected in self, the connection to the backend will be terminated, and early termination should always cause updates to fail on the backend. | | `eviction_policy` | [EvictionPolicy](#evictionpolicy) | — | — | Policy used to evict items out of the store. Failure to set this value will cause items to never be removed from the store causing infinite memory usage. | ## FastSlowSpec | Field | Type | Required | Default | Description | | --- | --- | --- | --- | --- | -| `fast` | [StoreSpec](#storespec) | Yes | — | Fast store that will be attempted to be contacted before reaching out to the `slow` store. | +| `fast` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | Fast store that will be attempted to be contacted before reaching out to the `slow` store. | | `fast_direction` | [StoreDirection](#storedirection) | — | `"both"` | How to handle the fast store. This can be useful to set to Get for worker nodes such that results are persisted to the slow store only. | -| `slow` | [StoreSpec](#storespec) | Yes | — | If the object does not exist in the `fast` store it will try to get it from this store. | +| `slow` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | If the object does not exist in the `fast` store it will try to get it from this store. | | `slow_direction` | [StoreDirection](#storedirection) | — | `"both"` | How to handle the slow store. This can be useful if creating a diode and you wish to have an upstream read only store. | | `bypass_dedup_threshold_bytes` | integer (uint64) | — | `0` | Reads of blobs at or above this size skip the leader/follower dedup map and stream straight from the slow store without populating the fast tier. `0` (the default) disables the bypass: every read goes through dedup, matching the prior behaviour. Enable it by setting a threshold — 256 MiB is a reasonable starting point for backends where large-blob dedup is a net loss (followers tend to time out anyway), but the right value is workload-dependent. | @@ -786,8 +786,8 @@ See [`experimental_cloud_object_store`](#experimental_cloud_object_store-1) for | Field | Type | Required | Default | Description | | --- | --- | --- | --- | --- | | `size` | integer (uint64) | Yes | — | Size to partition the data on. | -| `lower_store` | [StoreSpec](#storespec) | Yes | — | Store to send data when object is < (less than) size. | -| `upper_store` | [StoreSpec](#storespec) | Yes | — | Store to send data when object is >= (less than eq) size. | +| `lower_store` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | Store to send data when object is < (less than) size. | +| `upper_store` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | Store to send data when object is >= (less than eq) size. | ## GrpcSpec @@ -970,587 +970,6 @@ Listener for HTTP/HTTPS/HTTP2 sockets. | `"sha256"` | Use the sha256 hash function. [https://en.wikipedia.org/wiki/SHA-2](https://en.wikipedia.org/wiki/SHA-2) | | `"blake3"` | Use the blake3 hash function. [https://en.wikipedia.org/wiki/BLAKE_(hash_function)](https://en.wikipedia.org/wiki/BLAKE_(hash_function)) | -## StoreSpec - -Plus exactly one of the following variants (the key selects the variant): - -### `cache_metrics` - -Cache metrics store wraps another store and emits low-cardinality -OpenTelemetry cache operation metrics for the wrapped store. - -This wrapper is opt-in. Stores that are not explicitly wrapped by -`cache_metrics` are constructed exactly as they are without this -wrapper and do not pay its hot-path timing or recording cost. - -**Example JSON5 config:** -```json5 -"cache_metrics": { - "cache_type": "cas", - "backend": { - "filesystem": { - "content_path": "~/.cache/nativelink/content_path-cas", - "temp_path": "~/.cache/nativelink/tmp_path-cas" - } - } -} -``` - -**Type:** [CacheMetricsSpec](#cachemetricsspec) - -### `memory` - -Memory store will store all data in a hash map in memory. - -**Example JSON5 config:** -```json5 -"memory": { - "eviction_policy": { - "max_bytes": "10mb", - } -} -``` - -**Type:** [MemorySpec](#memoryspec) - -### `experimental_cloud_object_store` - -A generic blob store that will store files on the cloud -provider. This configuration will never delete files, so you are -responsible for purging old files in other ways. -It supports the following backends: - -1. **Amazon S3:** - S3 store will use Amazon's S3 service as a backend to store - the files. This configuration can be used to share files - across multiple instances. Uses system certificates for TLS - verification via `rustls-platform-verifier`. - - **Example JSON5 config:** - ```json5 - "experimental_cloud_object_store": { - "provider": "aws", - "region": "eu-north-1", - "bucket": "crossplane-bucket-af79aeca9", - "key_prefix": "test-prefix-index/", - "retry": { - "max_retries": 6, - "delay": 0.3, - "jitter": 0.5 - }, - "multipart_max_concurrent_uploads": 10 - } - ``` - -2. **Google Cloud Storage:** - GCS store uses Google's GCS service as a backend to store - the files. This configuration can be used to share files - across multiple instances. - - **Example JSON5 config:** - ```json5 - "experimental_cloud_object_store": { - "provider": "gcs", - "bucket": "test-bucket", - "key_prefix": "test-prefix-index/", - "retry": { - "max_retries": 6, - "delay": 0.3, - "jitter": 0.5 - }, - "multipart_max_concurrent_uploads": 10 - } - ``` - -3. **Azure Blob Store:** - Azure Blob store will use Microsoft's Azure Blob service as a - backend to store the files. This configuration can be used to - share files across multiple instances. - - **Example JSON5 config:** - ```json5 - "experimental_cloud_object_store": { - "provider": "azure", - "account_name": "cloudshell1393657559", - "container": "simple-test-container", - "key_prefix": "folder/", - "retry": { - "max_retries": 6, - "delay": 0.3, - "jitter": 0.5 - }, - "multipart_max_concurrent_uploads": 10 - } - ``` - -4. **`NetApp` ONTAP S3** - `NetApp` ONTAP S3 store will use ONTAP's S3-compatible storage as a backend - to store files. This store is specifically configured for ONTAP's S3 requirements - including custom TLS configuration, credentials management, and proper vserver - configuration. - - This store uses AWS environment variables for credentials: - - `AWS_ACCESS_KEY_ID` - - `AWS_SECRET_ACCESS_KEY` - - `AWS_DEFAULT_REGION` - - **Example JSON5 config:** - ```json5 - "experimental_cloud_object_store": { - "provider": "ontap", - "endpoint": "https://ontap-s3-endpoint:443", - "vserver_name": "your-vserver", - "bucket": "your-bucket", - "root_certificates": "/path/to/certs.pem", // Optional - "key_prefix": "test-prefix/", // Optional - "retry": { - "max_retries": 6, - "delay": 0.3, - "jitter": 0.5 - }, - "multipart_max_concurrent_uploads": 10 - } - ``` - -**Type:** [ExperimentalCloudObjectSpec](#experimentalcloudobjectspec) - -### `ontap_s3_existence_cache` - -ONTAP S3 Existence Cache provides a caching layer on top of the ONTAP S3 store -to optimize repeated existence checks. It maintains an in-memory cache of object -digests and periodically syncs this cache to disk for persistence. - -The cache helps reduce latency for repeated calls to check object existence, -while still ensuring eventual consistency with the underlying ONTAP S3 store. - -Example JSON5 config: -```json5 -"ontap_s3_existence_cache": { - "index_path": "/path/to/cache/index.json", - "sync_interval_seconds": 300, - "backend": { - "endpoint": "https://ontap-s3-endpoint:443", - "vserver_name": "your-vserver", - "bucket": "your-bucket", - "key_prefix": "test-prefix/" - } -} -``` - -**Type:** [OntapS3ExistenceCacheSpec](#ontaps3existencecachespec) - -### `verify` - -Verify store is used to apply verifications to an underlying -store implementation. It is strongly encouraged to validate -as much data as you can before accepting data from a client, -failing to do so may cause the data in the store to be -populated with invalid data causing all kinds of problems. - -The suggested configuration is to have the CAS validate the -hash and size and the AC validate nothing. - -**Example JSON5 config:** -```json5 -"verify": { - "backend": { - "memory": { - "eviction_policy": { - "max_bytes": "500mb" - } - }, - }, - "verify_size": true, - "verify_hash": true -} -``` - -**Type:** [VerifySpec](#verifyspec) - -### `completeness_checking` - -Completeness checking store verifies if the -output files & folders exist in the CAS before forwarding -the request to the underlying store. -Note: This store should only be used on AC stores. - -**Example JSON5 config:** -```json5 -"completeness_checking": { - "backend": { - "filesystem": { - "content_path": "~/.cache/nativelink/content_path-ac", - "temp_path": "~/.cache/nativelink/tmp_path-ac", - "eviction_policy": { - "max_bytes": "500mb", - } - } - }, - "cas_store": { - "ref_store": { - "name": "CAS_MAIN_STORE" - } - } -} -``` - -**Type:** [CompletenessCheckingSpec](#completenesscheckingspec) - -### `compression` - -A compression store that will compress the data inbound and -outbound. There will be a non-trivial cost to compress and -decompress the data, but in many cases if the final store is -a store that requires network transport and/or storage space -is a concern it is often faster and more efficient to use this -store before those stores. - -**Example JSON5 config:** -```json5 -"compression": { - "compression_algorithm": { - "lz4": {} - }, - "backend": { - "filesystem": { - "content_path": "/tmp/nativelink/data/content_path-cas", - "temp_path": "/tmp/nativelink/data/tmp_path-cas", - "eviction_policy": { - "max_bytes": "2gb", - } - } - } -} -``` - -**Type:** [CompressionSpec](#compressionspec) - -### `dedup` - -A dedup store will take the inputs and run a rolling hash -algorithm on them to slice the input into smaller parts then -run a sha256 algorithm on the slice and if the object doesn't -already exist, upload the slice to the `content_store` using -a new digest of just the slice. Once all parts exist, an -Action-Cache-like digest will be built and uploaded to the -`index_store` which will contain a reference to each -chunk/digest of the uploaded file. Downloading a request will -first grab the index from the `index_store`, and forward the -download content of each chunk as if it were one file. - -This store is exceptionally good when the following conditions -are met: -* Content is mostly the same (inserts, updates, deletes are ok) -* Content is not compressed or encrypted -* Uploading or downloading from `content_store` is the bottleneck. - -Note: This store pairs well when used with `CompressionSpec` as -the `content_store`, but never put `DedupSpec` as the backend of -`CompressionSpec` as it will negate all the gains. - -Note: When running `.has()` on this store, it will only check -to see if the entry exists in the `index_store` and not check -if the individual chunks exist in the `content_store`. - -**Example JSON5 config:** -```json5 -"dedup": { - "index_store": { - "memory": { - "eviction_policy": { - "max_bytes": "1GB", - } - } - }, - "content_store": { - "compression": { - "compression_algorithm": { - "lz4": {} - }, - "backend": { - "fast_slow": { - "fast": { - "memory": { - "eviction_policy": { - "max_bytes": "500MB", - } - } - }, - "slow": { - "filesystem": { - "content_path": "/tmp/nativelink/data/content_path-content", - "temp_path": "/tmp/nativelink/data/tmp_path-content", - "eviction_policy": { - "max_bytes": "2gb" - } - } - } - } - } - } - } -} -``` - -**Type:** [DedupSpec](#dedupspec) - -### `existence_cache` - -Existence store will wrap around another store and cache calls -to has so that subsequent `has_with_results` calls will be -faster. This is useful for cases when you have a store that -is slow to respond to has calls. -Note: This store should only be used on CAS stores. - -**Example JSON5 config:** -```json5 -"existence_cache": { - "backend": { - "memory": { - "eviction_policy": { - "max_bytes": "500mb", - } - } - }, - // Note this is the existence store policy, not the backend policy - "eviction_policy": { - "max_seconds": 100, - } -} -``` - -**Type:** [ExistenceCacheSpec](#existencecachespec) - -### `fast_slow` - -`FastSlow` store will first try to fetch the data from the `fast` -store and then if it does not exist try the `slow` store. -When the object does exist in the `slow` store, it will copy -the data to the `fast` store while returning the data. -This store should be thought of as a store that "buffers" -the data to the `fast` store. -On uploads it will mirror data to both `fast` and `slow` stores. - -WARNING: If you need data to always exist in the `slow` store -for something like remote execution, be careful because this -store will never check to see if the objects exist in the -`slow` store if it exists in the `fast` store (i.e., it assumes -that if an object exists in the `fast` store it will exist in -the `slow` store). - -***Example JSON5 config:*** -```json5 -"fast_slow": { - "fast": { - "filesystem": { - "content_path": "/tmp/nativelink/data/content_path-index", - "temp_path": "/tmp/nativelink/data/tmp_path-index", - "eviction_policy": { - "max_bytes": "500mb", - } - } - }, - "slow": { - "filesystem": { - "content_path": "/tmp/nativelink/data/content_path-index", - "temp_path": "/tmp/nativelink/data/tmp_path-index", - "eviction_policy": { - "max_bytes": "500mb", - } - } - } -} -``` - -**Type:** [FastSlowSpec](#fastslowspec) - -### `shard` - -Shards the data to multiple stores. This is useful for cases -when you want to distribute the load across multiple stores. -The digest hash is used to determine which store to send the -data to. - -**Example JSON5 config:** -```json5 -"shard": { - "stores": [ - { - "store": { - "memory": { - "eviction_policy": { - "max_bytes": "10mb" - }, - }, - }, - "weight": 1 - }] -} -``` - -**Type:** [ShardSpec](#shardspec) - -### `filesystem` - -Stores the data on the filesystem. This store is designed for -local persistent storage. Restarts of this program should restore -the previous state, meaning anything uploaded will be persistent -as long as the filesystem integrity holds. - -**Example JSON5 config:** -```json5 -"filesystem": { - "content_path": "/tmp/nativelink/data-worker-test/content_path-cas", - "temp_path": "/tmp/nativelink/data-worker-test/tmp_path-cas", - "eviction_policy": { - "max_bytes": "10gb", - } -} -``` - -**Type:** [FilesystemSpec](#filesystemspec) - -### `ref_store` - -Store used to reference a store in the root store manager. -This is useful for cases when you want to share a store in different -nested stores. Example, you may want to share the same memory store -used for the action cache, but use a `FastSlowSpec` and have the fast -store also share the memory store for efficiency. - -**Example JSON5 config:** -```json5 -"ref_store": { - "name": "FS_CONTENT_STORE" -} -``` - -**Type:** [RefSpec](#refspec) - -### `size_partitioning` - -Uses the size field of the digest to separate which store to send the -data. This is useful for cases when you'd like to put small objects -in one store and large objects in another store. This should only be -used if the size field is the real size of the content, in other -words, don't use on AC (Action Cache) stores. Any store where you can -safely use `VerifySpec.verify_size = true`, this store should be safe -to use (i.e., CAS stores). - -**Example JSON5 config:** -```json5 -"size_partitioning": { - "size": "128mib", - "lower_store": { - "memory": { - "eviction_policy": { - "max_bytes": "${NATIVELINK_CAS_MEMORY_CONTENT_LIMIT:-100mb}" - } - } - }, - "upper_store": { - /// This store discards data larger than 128mib. - "noop": {} - } -} -``` - -**Type:** [SizePartitioningSpec](#sizepartitioningspec) - -### `grpc` - -This store will pass-through calls to another GRPC store. This store -is not designed to be used as a sub-store of another store, but it -does satisfy the interface and will likely work. - -One major GOTCHA is that some stores use a special function on this -store to get the size of the underlying object, which is only reliable -when this store is serving the a CAS store, not an AC store. If using -this store directly without being a child of any store there are no -side effects and is the most efficient way to use it. - -**Example JSON5 config:** -```json5 -"grpc": { - "instance_name": "main", - "endpoints": [ - {"address": "grpc://${CAS_ENDPOINT:-127.0.0.1}:50051"} - ], - "connections_per_endpoint": "5", - "rpc_timeout_s": "5m", - "store_type": "ac", - // Static headers attached to every outgoing request to the upstream - // remote cache. Useful for fixed service-account credentials. - "headers": { - "authorization": "Bearer my-static-token" - }, - // Header names to copy from the inbound client request and forward to - // the upstream remote cache. Use this to pass through dynamic - // credentials such as a JWT sent by the build client. - "forward_headers": ["authorization", "x-custom-token"] -} -``` - -**Type:** [GrpcSpec](#grpcspec) - -### `redis_store` - -Stores data in any stores compatible with Redis APIs. - -Pairs well with `SizePartitioning` and/or `FastSlow` stores. -Ideal for accepting small object sizes as most Redis store -services have a max file upload of between 256Mb-512Mb. - -**Example JSON5 config:** -```json5 -"redis_store": { - "addresses": [ - "redis://127.0.0.1:6379/", - ], - "max_client_permits": 1000, -} -``` - -**Type:** [RedisSpec](#redisspec) - -### `noop` - -Noop store is a store that sends streams into the void and all data -retrieval will return 404 (`NotFound`). This can be useful for cases -where you may need to partition your data and part of your data needs -to be discarded. - -**Example JSON5 config:** -```json5 -"noop": {} -``` - -**Type:** [NoopSpec](#noopspec) - -### `experimental_mongo` - -Experimental `MongoDB` store implementation. - -This store uses `MongoDB` as a backend for storing data. It supports -both CAS (Content Addressable Storage) and scheduler data with -optional change streams for real-time updates. - -**Example JSON5 config:** -```json5 -"experimental_mongo": { - "connection_string": "mongodb://localhost:27017", - "database": "nativelink", - "cas_collection": "cas", - "key_prefix": "cas:", - "read_chunk_size": 65536, - "max_concurrent_uploads": 10, - "enable_change_streams": false, - "max_requests": "100" -} -``` - -**Type:** [ExperimentalMongoSpec](#experimentalmongospec) - ## EvictionPolicy Eviction policy always works on LRU (Least Recently Used). Any time an entry @@ -1645,7 +1064,7 @@ Configuration for an individual shard of the store. | Field | Type | Required | Default | Description | | --- | --- | --- | --- | --- | -| `store` | [StoreSpec](#storespec) | Yes | — | Store to shard the data to. | +| `store` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | Store to shard the data to. | | `weight` | integer (uint32) | — | 1 | The weight of the store. This is used to determine how much data should be sent to the store. The actual percentage is the sum of all the store's weights divided by the individual store's weight. | ## GrpcEndpoint diff --git a/web/apps/docs/content/docs/reference/nativelink-config/v1.5.2.mdx b/web/apps/docs/content/docs/reference/nativelink-config/v1.5.2.mdx new file mode 100644 index 000000000..e57c2537b --- /dev/null +++ b/web/apps/docs/content/docs/reference/nativelink-config/v1.5.2.mdx @@ -0,0 +1,1460 @@ +--- +title: Configuration reference +description: Every knob in the NativeLink JSON5 configuration — types, defaults, and links to source, autogenerated from the Rust config crate. +full: true +--- + +{/* AUTOGENERATED — do not edit by hand. + Source: nativelink-config @ v1.5.2 (6e63ef9a) + Regenerate from web/: bun --filter @nativelink/docs gen:config-reference */} + + + +This is the canonical NativeLink configuration reference for **v1.5.2**. +It is autogenerated from the Rust config crate +([`nativelink-config/src`](https://github.com/TraceMachina/nativelink/tree/v1.5.2/nativelink-config/src)) via the `build-schema` binary, so +it can never drift from what the binary actually deserializes. + +## Top-level fields + +The root object (`CasConfig`) accepts the following fields: + +| Field | Type | Required | Description | +| --- | --- | --- | --- | +| `stores` | array of [NamedStoreConfig](#namedstoreconfig) | Yes | List of stores available to use in this config. The keys can be used in other configs when needing to reference a store. | +| `workers` | array of [WorkerConfig](#workerconfig) | — | Worker configurations used to execute jobs. | +| `schedulers` | array of [NamedSchedulerConfig](#namedschedulerconfig) | — | List of schedulers available to use in this config. The keys can be used in other configs when needing to reference a scheduler. | +| `servers` | array of [ServerConfig](#serverconfig) | Yes | Servers to setup for this process. | +| `experimental_origin_events` | [OriginEventsSpec](#origineventsspec) | — | Experimental - Origin events configuration. This is the service that will collect and publish nativelink events to a store for processing by an external service. | +| `global` | [GlobalConfig](#globalconfig) | — | Any global configurations that apply to all modules live here. | + +## Configuration types + +Every type reachable from the root configuration, in reading order. + +## NamedStoreConfig + +**Common fields** + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `name` | string | Yes | — | | + +Plus exactly one of the following variants (the key selects the variant): + +### `cache_metrics` + +Cache metrics store wraps another store and emits low-cardinality +OpenTelemetry cache operation metrics for the wrapped store. + +This wrapper is opt-in. Stores that are not explicitly wrapped by +`cache_metrics` are constructed exactly as they are without this +wrapper and do not pay its hot-path timing or recording cost. + +**Example JSON5 config:** +```json5 +"cache_metrics": { + "cache_type": "cas", + "backend": { + "filesystem": { + "content_path": "~/.cache/nativelink/content_path-cas", + "temp_path": "~/.cache/nativelink/tmp_path-cas" + } + } +} +``` + +**Type:** [CacheMetricsSpec](#cachemetricsspec) + +### `memory` + +Memory store will store all data in a hash map in memory. + +**Example JSON5 config:** +```json5 +"memory": { + "eviction_policy": { + "max_bytes": "10mb", + } +} +``` + +**Type:** [MemorySpec](#memoryspec) + +### `experimental_cloud_object_store` + +A generic blob store that will store files on the cloud +provider. This configuration will never delete files, so you are +responsible for purging old files in other ways. +It supports the following backends: + +1. **Amazon S3:** + S3 store will use Amazon's S3 service as a backend to store + the files. This configuration can be used to share files + across multiple instances. Uses system certificates for TLS + verification via `rustls-platform-verifier`. + + **Example JSON5 config:** + ```json5 + "experimental_cloud_object_store": { + "provider": "aws", + "region": "eu-north-1", + "bucket": "crossplane-bucket-af79aeca9", + "key_prefix": "test-prefix-index/", + "retry": { + "max_retries": 6, + "delay": 0.3, + "jitter": 0.5 + }, + "multipart_max_concurrent_uploads": 10 + } + ``` + +2. **Google Cloud Storage:** + GCS store uses Google's GCS service as a backend to store + the files. This configuration can be used to share files + across multiple instances. + + **Example JSON5 config:** + ```json5 + "experimental_cloud_object_store": { + "provider": "gcs", + "bucket": "test-bucket", + "key_prefix": "test-prefix-index/", + "retry": { + "max_retries": 6, + "delay": 0.3, + "jitter": 0.5 + }, + "multipart_max_concurrent_uploads": 10 + } + ``` + +3. **Azure Blob Store:** + Azure Blob store will use Microsoft's Azure Blob service as a + backend to store the files. This configuration can be used to + share files across multiple instances. + + **Example JSON5 config:** + ```json5 + "experimental_cloud_object_store": { + "provider": "azure", + "account_name": "cloudshell1393657559", + "container": "simple-test-container", + "key_prefix": "folder/", + "retry": { + "max_retries": 6, + "delay": 0.3, + "jitter": 0.5 + }, + "multipart_max_concurrent_uploads": 10 + } + ``` + +4. **NetApp ONTAP S3:** + NetApp ONTAP S3 store will use ONTAP's S3-compatible storage as a backend + to store files. This store is specifically configured for ONTAP's S3 requirements + including custom TLS configuration, credentials management, and proper vserver + configuration. + + This store uses AWS environment variables for credentials: + - `AWS_ACCESS_KEY_ID` + - `AWS_SECRET_ACCESS_KEY` + - `AWS_DEFAULT_REGION` + + **Example JSON5 config:** + ```json5 + "experimental_cloud_object_store": { + "provider": "ontap", + "endpoint": "https://ontap-s3-endpoint:443", + "vserver_name": "your-vserver", + "bucket": "your-bucket", + "root_certificates": "/path/to/certs.pem", // Optional + "key_prefix": "test-prefix/", // Optional + "retry": { + "max_retries": 6, + "delay": 0.3, + "jitter": 0.5 + }, + "multipart_max_concurrent_uploads": 10 + } + ``` + +**Type:** [ExperimentalCloudObjectSpec](#experimentalcloudobjectspec) + +### `ontap_s3_existence_cache` + +ONTAP S3 Existence Cache provides a caching layer on top of the ONTAP S3 store +to optimize repeated existence checks. It maintains an in-memory cache of object +digests and periodically syncs this cache to disk for persistence. + +The cache helps reduce latency for repeated calls to check object existence, +while still ensuring eventual consistency with the underlying ONTAP S3 store. + +Example JSON5 config: +```json5 +"ontap_s3_existence_cache": { + "index_path": "/path/to/cache/index.json", + "sync_interval_seconds": 300, + "backend": { + "endpoint": "https://ontap-s3-endpoint:443", + "vserver_name": "your-vserver", + "bucket": "your-bucket", + "key_prefix": "test-prefix/" + } +} +``` + +**Type:** [OntapS3ExistenceCacheSpec](#ontaps3existencecachespec) + +### `verify` + +Verify store is used to apply verifications to an underlying +store implementation. It is strongly encouraged to validate +as much data as you can before accepting data from a client, +failing to do so may cause the data in the store to be +populated with invalid data causing all kinds of problems. + +The suggested configuration is to have the CAS validate the +hash and size and the AC validate nothing. + +**Example JSON5 config:** +```json5 +"verify": { + "backend": { + "memory": { + "eviction_policy": { + "max_bytes": "500mb" + } + }, + }, + "verify_size": true, + "verify_hash": true +} +``` + +**Type:** [VerifySpec](#verifyspec) + +### `completeness_checking` + +Completeness checking store verifies if the +output files & folders exist in the CAS before forwarding +the request to the underlying store. +Note: This store should only be used on AC stores. + +**Example JSON5 config:** +```json5 +"completeness_checking": { + "backend": { + "filesystem": { + "content_path": "~/.cache/nativelink/content_path-ac", + "temp_path": "~/.cache/nativelink/tmp_path-ac", + "eviction_policy": { + "max_bytes": "500mb", + } + } + }, + "cas_store": { + "ref_store": { + "name": "CAS_MAIN_STORE" + } + } +} +``` + +**Type:** [CompletenessCheckingSpec](#completenesscheckingspec) + +### `compression` + +A compression store that will compress the data inbound and +outbound. There will be a non-trivial cost to compress and +decompress the data, but in many cases if the final store is +a store that requires network transport and/or storage space +is a concern it is often faster and more efficient to use this +store before those stores. + +**Example JSON5 config:** +```json5 +"compression": { + "compression_algorithm": { + "lz4": {} + }, + "backend": { + "filesystem": { + "content_path": "/tmp/nativelink/data/content_path-cas", + "temp_path": "/tmp/nativelink/data/tmp_path-cas", + "eviction_policy": { + "max_bytes": "2gb", + } + } + } +} +``` + +**Type:** [CompressionSpec](#compressionspec) + +### `dedup` + +A dedup store will take the inputs and run a rolling hash +algorithm on them to slice the input into smaller parts then +run a sha256 algorithm on the slice and if the object doesn't +already exist, upload the slice to the `content_store` using +a new digest of just the slice. Once all parts exist, an +Action-Cache-like digest will be built and uploaded to the +`index_store` which will contain a reference to each +chunk/digest of the uploaded file. Downloading a request will +first grab the index from the `index_store`, and forward the +download content of each chunk as if it were one file. + +This store is exceptionally good when the following conditions +are met: +* Content is mostly the same (inserts, updates, deletes are ok) +* Content is not compressed or encrypted +* Uploading or downloading from `content_store` is the bottleneck. + +Note: This store pairs well when used with `CompressionSpec` as +the `content_store`, but never put `DedupSpec` as the backend of +`CompressionSpec` as it will negate all the gains. + +Note: When running `.has()` on this store, it will only check +to see if the entry exists in the `index_store` and not check +if the individual chunks exist in the `content_store`. + +**Example JSON5 config:** +```json5 +"dedup": { + "index_store": { + "memory": { + "eviction_policy": { + "max_bytes": "1GB", + } + } + }, + "content_store": { + "compression": { + "compression_algorithm": { + "lz4": {} + }, + "backend": { + "fast_slow": { + "fast": { + "memory": { + "eviction_policy": { + "max_bytes": "500MB", + } + } + }, + "slow": { + "filesystem": { + "content_path": "/tmp/nativelink/data/content_path-content", + "temp_path": "/tmp/nativelink/data/tmp_path-content", + "eviction_policy": { + "max_bytes": "2gb" + } + } + } + } + } + } + } +} +``` + +**Type:** [DedupSpec](#dedupspec) + +### `existence_cache` + +Existence store will wrap around another store and cache calls +to has so that subsequent `has_with_results` calls will be +faster. This is useful for cases when you have a store that +is slow to respond to has calls. +Note: This store should only be used on CAS stores. + +**Example JSON5 config:** +```json5 +"existence_cache": { + "backend": { + "memory": { + "eviction_policy": { + "max_bytes": "500mb", + } + } + }, + // Note this is the existence store policy, not the backend policy + "eviction_policy": { + "max_seconds": 100, + } +} +``` + +**Type:** [ExistenceCacheSpec](#existencecachespec) + +### `fast_slow` + +`FastSlow` store will first try to fetch the data from the `fast` +store and then if it does not exist try the `slow` store. +When the object does exist in the `slow` store, it will copy +the data to the `fast` store while returning the data. +This store should be thought of as a store that "buffers" +the data to the `fast` store. +On uploads it will mirror data to both `fast` and `slow` stores. + +WARNING: If you need data to always exist in the `slow` store +for something like remote execution, be careful because this +store will never check to see if the objects exist in the +`slow` store if it exists in the `fast` store (i.e., it assumes +that if an object exists in the `fast` store it will exist in +the `slow` store). + +***Example JSON5 config:*** +```json5 +"fast_slow": { + "fast": { + "filesystem": { + "content_path": "/tmp/nativelink/data/content_path-index", + "temp_path": "/tmp/nativelink/data/tmp_path-index", + "eviction_policy": { + "max_bytes": "500mb", + } + } + }, + "slow": { + "filesystem": { + "content_path": "/tmp/nativelink/data/content_path-index", + "temp_path": "/tmp/nativelink/data/tmp_path-index", + "eviction_policy": { + "max_bytes": "500mb", + } + } + } +} +``` + +**Type:** [FastSlowSpec](#fastslowspec) + +### `shard` + +Shards the data to multiple stores. This is useful for cases +when you want to distribute the load across multiple stores. +The digest hash is used to determine which store to send the +data to. + +**Example JSON5 config:** +```json5 +"shard": { + "stores": [ + { + "store": { + "memory": { + "eviction_policy": { + "max_bytes": "10mb" + }, + }, + }, + "weight": 1 + }] +} +``` + +**Type:** [ShardSpec](#shardspec) + +### `filesystem` + +Stores the data on the filesystem. This store is designed for +local persistent storage. Restarts of this program should restore +the previous state, meaning anything uploaded will be persistent +as long as the filesystem integrity holds. + +**Example JSON5 config:** +```json5 +"filesystem": { + "content_path": "/tmp/nativelink/data-worker-test/content_path-cas", + "temp_path": "/tmp/nativelink/data-worker-test/tmp_path-cas", + "eviction_policy": { + "max_bytes": "10gb", + } +} +``` + +**Type:** [FilesystemSpec](#filesystemspec) + +### `ref_store` + +Store used to reference a store in the root store manager. +This is useful for cases when you want to share a store in different +nested stores. Example, you may want to share the same memory store +used for the action cache, but use a `FastSlowSpec` and have the fast +store also share the memory store for efficiency. + +**Example JSON5 config:** +```json5 +"ref_store": { + "name": "FS_CONTENT_STORE" +} +``` + +**Type:** [RefSpec](#refspec) + +### `size_partitioning` + +Uses the size field of the digest to separate which store to send the +data. This is useful for cases when you'd like to put small objects +in one store and large objects in another store. This should only be +used if the size field is the real size of the content, in other +words, don't use on AC (Action Cache) stores. Any store where you can +safely use `VerifySpec.verify_size = true`, this store should be safe +to use (i.e., CAS stores). + +**Example JSON5 config:** +```json5 +"size_partitioning": { + "size": "128mib", + "lower_store": { + "memory": { + "eviction_policy": { + "max_bytes": "${NATIVELINK_CAS_MEMORY_CONTENT_LIMIT:-100mb}" + } + } + }, + "upper_store": { + /// This store discards data larger than 128mib. + "noop": {} + } +} +``` + +**Type:** [SizePartitioningSpec](#sizepartitioningspec) + +### `grpc` + +This store will pass-through calls to another GRPC store. This store +is not designed to be used as a sub-store of another store, but it +does satisfy the interface and will likely work. + +One major GOTCHA is that some stores use a special function on this +store to get the size of the underlying object, which is only reliable +when this store is serving the a CAS store, not an AC store. If using +this store directly without being a child of any store there are no +side effects and is the most efficient way to use it. + +**Example JSON5 config:** +```json5 +"grpc": { + "instance_name": "main", + "endpoints": [ + {"address": "grpc://${CAS_ENDPOINT:-127.0.0.1}:50051"} + ], + "connections_per_endpoint": "5", + "rpc_timeout_s": "5m", + "store_type": "ac", + // Static headers attached to every outgoing request to the upstream + // remote cache. Useful for fixed service-account credentials. + "headers": { + "authorization": "Bearer my-static-token" + }, + // Header names to copy from the inbound client request and forward to + // the upstream remote cache. Use this to pass through dynamic + // credentials such as a JWT sent by the build client. + "forward_headers": ["authorization", "x-custom-token"] +} +``` + +**Type:** [GrpcSpec](#grpcspec) + +### `redis_store` + +Stores data in any stores compatible with Redis APIs. + +Pairs well with `SizePartitioning` and/or `FastSlow` stores. +Ideal for accepting small object sizes as most Redis store +services have a max file upload of between 256Mb-512Mb. + +**Example JSON5 config:** +```json5 +"redis_store": { + "addresses": [ + "redis://127.0.0.1:6379/", + ], + "max_client_permits": 1000, +} +``` + +**Type:** [RedisSpec](#redisspec) + +### `noop` + +Noop store is a store that sends streams into the void and all data +retrieval will return 404 (`NotFound`). This can be useful for cases +where you may need to partition your data and part of your data needs +to be discarded. + +**Example JSON5 config:** +```json5 +"noop": {} +``` + +**Type:** [NoopSpec](#noopspec) + +### `experimental_mongo` + +Experimental `MongoDB` store implementation. + +This store uses `MongoDB` as a backend for storing data. It supports +both CAS (Content Addressable Storage) and scheduler data with +optional change streams for real-time updates. + +**Example JSON5 config:** +```json5 +"experimental_mongo": { + "connection_string": "mongodb://localhost:27017", + "database": "nativelink", + "cas_collection": "cas", + "key_prefix": "cas:", + "read_chunk_size": 65536, + "max_concurrent_uploads": 10, + "enable_change_streams": false, + "max_requests": "100" +} +``` + +**Type:** [ExperimentalMongoSpec](#experimentalmongospec) + +## WorkerConfig + +Plus exactly one of the following variants (the key selects the variant): + +### `local` + +A worker type that executes jobs locally on this machine. + +**Type:** [LocalWorkerConfig](#localworkerconfig) + +## NamedSchedulerConfig + +**Common fields** + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `name` | string | Yes | — | | + +Plus exactly one of the following variants (the key selects the variant): + +### `simple` + +**Type:** [SimpleSpec](#simplespec) + +### `grpc` + +**Type:** [SchedulerGrpcSpec](#schedulergrpcspec) + +### `cache_lookup` + +**Type:** [CacheLookupSpec](#cachelookupspec) + +### `property_modifier` + +**Type:** [PropertyModifierSpec](#propertymodifierspec) + +### `historical_resource` + +**Type:** [HistoricalResourceSpec](#historicalresourcespec) + +## ServerConfig + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `name` | string | — | {index of server in config} | Name of the server. This is used to help identify the service for telemetry and logs. | +| `listener` | [ListenerConfig](#listenerconfig) | Yes | — | Configuration | +| `services` | [ServicesConfig](#servicesconfig) | — | — | Services to attach to server. | +| `experimental_identity_header` | [IdentityHeaderSpec](#identityheaderspec) | — | {see `IdentityHeaderSpec`} | The config related to identifying the client. | + +## OriginEventsSpec + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `publisher` | [OriginEventsPublisherSpec](#origineventspublisherspec) | Yes | — | The publisher configuration for origin events. | +| `max_event_queue_size` | integer (uint) | — | 65536 (zero defaults to this) | The maximum number of events to queue before applying back pressure. IMPORTANT: Backpressure causes all clients to slow down significantly. Zero is default. | + +## GlobalConfig + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `max_open_files` | integer (uint) | Yes | 24576 (= 24 * 1024) | Maximum number of open files that can be opened at one time. This value is not strictly enforced, it is a best effort. Some internal libraries open files or read metadata from a files which do not obey this limit, however the vast majority of cases will have this limit be honored. This value must be larger than `ulimit -n` to have any effect. Any network open file descriptors is not counted in this limit, but is counted in the kernel limit. It is a good idea to set a very large `ulimit -n`. Note: This value must be greater than 10. | +| `default_digest_hash_function` | [ConfigDigestHashFunction](#configdigesthashfunction) | — | `ConfigDigestHashFunction::sha256` | Default hash function to use while uploading blobs to the CAS when not set by client. | +| `default_digest_size_health_check` | integer (uint) | — | 1024*1024 (1MiB) | Default digest size to use for health check when running diagnostics checks. Health checks are expected to use this size for filling a buffer that is used for creation of digest. | + +## CacheMetricsSpec + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `cache_type` | string | Yes | — | Low-cardinality cache type label for metrics, for example `cas` or `ac`. | +| `backend` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | Store to wrap with cache operation metrics. | + +## MemorySpec + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `eviction_policy` | [EvictionPolicy](#evictionpolicy) | — | — | Policy used to evict items out of the store. Failure to set this value will cause items to never be removed from the store causing infinite memory usage. | + +## ExperimentalCloudObjectSpec + +See [`experimental_cloud_object_store`](#experimental_cloud_object_store-1) for details + +## OntapS3ExistenceCacheSpec + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `index_path` | string | Yes | — | | +| `sync_interval_seconds` | integer (uint32) | Yes | — | | +| `backend` | [ExperimentalOntapS3Spec](#experimentalontaps3spec) | Yes | — | | + +## VerifySpec + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `backend` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | The underlying store wrap around. All content will first flow through self before forwarding to backend. In the event there is an error detected in self, the connection to the backend will be terminated, and early termination should always cause updates to fail on the backend. | +| `verify_size` | boolean | — | `false` | If set the store will verify the size of the data before accepting an upload of data. | +| `verify_hash` | boolean | — | `false` | If the data should be hashed and verify that the key matches the computed hash. The hash function is automatically determined based request and if not set will use the global default. | + +## CompletenessCheckingSpec + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `backend` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | The underlying store that will have it's results validated before sending to client. | +| `cas_store` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | When a request is made, the results are decoded and all output digests/files are verified to exist in this CAS store before returning success. | + +## CompressionSpec + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `backend` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | The underlying store wrap around. All content will first flow through self before forwarding to backend. In the event there is an error detected in self, the connection to the backend will be terminated, and early termination should always cause updates to fail on the backend. | +| `compression_algorithm` | [CompressionAlgorithm](#compressionalgorithm) | Yes | — | The compression algorithm to use. | + +## DedupSpec + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `index_store` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | Store used to store the index of each dedup slice. This store should generally be fast and small. | +| `content_store` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | The store where the individual chunks will be uploaded. This store should generally be the slower & larger store. | +| `min_size` | integer (uint32) | — | 64k | Minimum size that a chunk will be when slicing up the content. Note: This setting can be increased to improve performance because it will actually not check this number of bytes when deciding where to partition the data. | +| `normal_size` | integer (uint32) | — | 256k | A best-effort attempt will be made to keep the average size of the chunks to this number. It is not a guarantee, but a slight attempt will be made. | +| `max_size` | integer (uint32) | — | 512k | Maximum size a chunk is allowed to be. | +| `max_concurrent_fetch_per_get` | integer (uint32) | — | 10 | Due to implementation detail, we want to prefer to download the first chunks of the file so we can stream the content out and free up some of our buffers. This configuration will be used to restrict the number of concurrent chunk downloads at a time per `get()` request. | + +## ExistenceCacheSpec + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `backend` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | The underlying store wrap around. All content will first flow through self before forwarding to backend. In the event there is an error detected in self, the connection to the backend will be terminated, and early termination should always cause updates to fail on the backend. | +| `eviction_policy` | [EvictionPolicy](#evictionpolicy) | — | — | Policy used to evict items out of the store. Failure to set this value will cause items to never be removed from the store causing infinite memory usage. | + +## FastSlowSpec + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `fast` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | Fast store that will be attempted to be contacted before reaching out to the `slow` store. | +| `fast_direction` | [StoreDirection](#storedirection) | — | `"both"` | How to handle the fast store. This can be useful to set to Get for worker nodes such that results are persisted to the slow store only. | +| `slow` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | If the object does not exist in the `fast` store it will try to get it from this store. | +| `slow_direction` | [StoreDirection](#storedirection) | — | `"both"` | How to handle the slow store. This can be useful if creating a diode and you wish to have an upstream read only store. | +| `bypass_dedup_threshold_bytes` | integer (uint64) | — | disabled (0) | Reads of blobs at or above this size skip the leader/follower dedup map and stream straight from the slow store without populating the fast tier. `0` (the default) disables the bypass: every read goes through dedup, matching the prior behaviour. Enable it by setting a threshold — 256 MiB is a reasonable starting point for backends where large-blob dedup is a net loss (followers tend to time out anyway), but the right value is workload-dependent. | + +## ShardSpec + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `stores` | array of [ShardConfig](#shardconfig) | Yes | — | Stores to shard the data to. | + +## FilesystemSpec + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `content_path` | string | Yes | — | Path on the system where to store the actual content. This is where the bulk of the data will be placed. On service startup this folder will be scanned and all files will be added to the cache. In the event one of the files doesn't match the criteria, the file will be deleted. | +| `temp_path` | string | Yes | — | A temporary location of where files that are being uploaded or deleted will be placed while the content cannot be guaranteed to be accurate. This location must be on the same block device as `content_path` so atomic moves can happen (i.e., move without copy). All files in this folder will be deleted on every startup. | +| `read_buffer_size` | integer (uint32) | — | 32k | Buffer size to use when reading files. Generally this should be left to the default value except for testing. | +| `eviction_policy` | [EvictionPolicy](#evictionpolicy) | — | — | Policy used to evict items out of the store. Failure to set this value will cause items to never be removed from the store causing infinite memory usage. | +| `block_size` | integer (uint64) | — | 4kb | The block size of the filesystem for the running machine value is used to determine an entry's actual size on disk consumed For a 4KB block size filesystem, a 1B file actually consumes 4KB | +| `max_concurrent_writes` | integer (uint) | — | unlimited | Maximum number of concurrent write operations allowed. Each write involves streaming data to a temp file and calling `sync_all()`, which can saturate disk I/O when many writes happen simultaneously. Limiting concurrency prevents disk saturation from blocking the async runtime. A value of 0 means unlimited (no concurrency limit). | + +## RefSpec + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `name` | string | Yes | — | Name of the store under the root "stores" config object. | + +## SizePartitioningSpec + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `size` | integer (uint64) | Yes | — | Size to partition the data on. | +| `lower_store` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | Store to send data when object is < (less than) size. | +| `upper_store` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | Store to send data when object is >= (less than eq) size. | + +## GrpcSpec + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `instance_name` | string | — | `""` | Instance name for GRPC calls. Proxy calls will have the `instance_name` changed to this. | +| `endpoints` | array of [GrpcEndpoint](#grpcendpoint) | Yes | — | The endpoint of the grpc connection. | +| `store_type` | [StoreType](#storetype) | Yes | — | The type of the upstream store, this ensures that the correct server calls are made. | +| `retry` | [Retry](#retry) | — | — | Retry configuration to use when a network request fails. | +| `max_concurrent_requests` | integer (uint) | — | `0` | Limit the number of simultaneous upstream requests to this many. A value of zero is treated as unlimited. If the limit is reached the request is queued. | +| `connections_per_endpoint` | integer (uint) | — | `0` | The number of connections to make to each specified endpoint to balance the load over multiple TCP connections. Default 1. | +| `rpc_timeout_s` | integer (uint64) | — | 0 (disabled) | Maximum time (seconds) allowed for a single RPC request (e.g. a `ByteStream.Write` call) before it is cancelled. | +| `use_legacy_resource_names` | boolean | — | false | Use legacy `ByteStream` resource name format, omitting the digest function component from the path. | +| `headers` | map of string to string | — | — | Static headers to attach to every outgoing gRPC request sent to this store's upstream endpoints. Useful for fixed authentication tokens (e.g. `{"authorization": "Bearer "}`) and other static metadata. | +| `forward_headers` | array of string | — | — | Header names to forward from the incoming client request to every outgoing upstream request. The header value is taken from the client request that triggered this store operation. Use this to pass through dynamic credentials such as JWT tokens sent by build clients. | + +## RedisSpec + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `addresses` | array of string | Yes | — | The hostname or IP address of the Redis server. Ex: `["redis://username:password@redis-server-url:6380/99"]` 99 Represents database ID, 6380 represents the port. | +| `response_timeout_s` | integer (uint64) | — | 10 | DEPRECATED: use `command_timeout_ms` The response timeout for the Redis connection in seconds. | +| `connection_timeout_s` | integer (uint64) | — | 10 | DEPRECATED: use `connection_timeout_ms` | +| `experimental_pub_sub_channel` | string | — | (Empty String / No Channel) | An optional and experimental Redis channel to publish write events to. | +| `key_prefix` | string | — | (Empty String / No Prefix) | An optional prefix to prepend to all keys in this store. | +| `mode` | [RedisMode](#redismode) | — | standard, | Set the mode Redis is operating in. | +| `broadcast_channel_capacity` | integer (uint) | — | `0` | Deprecated as redis-rs doesn't use it | +| `command_timeout_ms` | integer (uint64) | — | 10000 (10 seconds) | The amount of time in milliseconds until the Redis store considers the command to be timed out. This will trigger a retry of the command and potentially a reconnection to the Redis server. | +| `connection_timeout_ms` | integer (uint64) | — | 3000 (3 seconds) | The amount of time in milliseconds until the Redis store considers the connection to unresponsive. This will trigger a reconnection to the Redis server. | +| `health_check_timeout_ms` | integer (uint64) | — | 4000 (4 seconds) | Per-call ceiling for the `check_health` PING in milliseconds. | +| `read_chunk_size` | integer (uint) | — | 64KiB | The amount of data to read from the Redis server at a time. This is used to limit the amount of memory used when reading large objects from the Redis server as well as limiting the amount of time a single read operation can take. | +| `connection_pool_size` | integer (uint) | — | 3 | The number of connections to keep open to the Redis servers. | +| `max_chunk_uploads_per_update` | integer (uint) | — | 10 | The maximum number of upload chunks to allow per update. This is used to limit the amount of memory used when uploading large objects to the Redis server. A good rule of thumb is to think of the data as: `AVAIL_MEMORY / (read_chunk_size * max_chunk_uploads_per_update) = THORETICAL_MAX_CONCURRENT_UPLOADS` (note: it is a good idea to divide `AVAIL_MAX_MEMORY` by ~10 to account for other memory usage) | +| `scan_count` | integer (uint) | — | 10000 | The COUNT value passed when scanning keys in Redis. This is used to hint the amount of work that should be done per response. | +| `retry` | [Retry](#retry) | — | — | Retry configuration to use when a network request fails. | +| `max_client_permits` | integer (uint) | — | 500 | Maximum number of permitted actions to the Redis store at any one time This stops problems with timeouts due to many, many inflight actions | +| `max_count_per_cursor` | integer (uint64) | — | 1500 | Maximum number of items returned per cursor for the search indexes May reduce thundering herd issues with worker provisioner at higher node counts, | + +## NoopSpec + +_No fields._ + +## ExperimentalMongoSpec + +Configuration for `ExperimentalMongoDB` store. + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `connection_string` | string | Yes | — | `ExperimentalMongoDB` connection string. Example: <mongodb://localhost:27017> or <mongodb+srv://cluster.mongodb.net> | +| `database` | string | — | "nativelink" | The database name to use. | +| `cas_collection` | string | — | "cas" | The collection name for CAS data. | +| `scheduler_collection` | string | — | "scheduler" | The collection name for scheduler data. | +| `key_prefix` | string | — | "" | Prefix to prepend to all keys stored in `MongoDB`. | +| `read_chunk_size` | integer (uint) | — | 65536 (64KB) | The maximum amount of data to read from `MongoDB` in a single chunk (in bytes). | +| `max_concurrent_uploads` | integer (uint) | — | 10 | Deprecated, unused Maximum number of concurrent uploads allowed. | +| `connection_timeout_ms` | integer (uint64) | — | 3000 | Connection timeout in milliseconds. | +| `command_timeout_ms` | integer (uint64) | — | 10000 | Command timeout in milliseconds. | +| `enable_change_streams` | boolean | — | false | Enable `MongoDB` change streams for real-time updates. Required for scheduler subscriptions. | +| `write_concern_w` | string | — | — | Write concern 'w' parameter. Can be a number (e.g., 1) or string (e.g., "majority"). | +| `write_concern_j` | boolean | — | — | Write concern 'j' parameter (journal acknowledgment). | +| `write_concern_timeout_ms` | integer (uint32) | — | — | Write concern timeout in milliseconds. | +| `max_requests` | integer (uint) | — | Unlimited | Limits the number of requests at any one time | + +## LocalWorkerConfig + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `name` | string | — | {Index position in the workers list} | Name of the worker. This is give a more friendly name to a worker for logging and metric publishing. This is also the prefix of the worker id (i.e., "{name}{uuidv6}"). | +| `worker_api_endpoint` | [EndpointConfig](#endpointconfig) | Yes | — | Endpoint which the worker will connect to the scheduler's `WorkerApiService`. | +| `max_action_timeout` | integer (uint) | — | 20 minutes | The maximum time an action is allowed to run. If a task requests for a timeout longer than this time limit, the task will be rejected. Value in seconds. | +| `max_upload_timeout` | integer (uint) | — | 10 minutes | Maximum time allowed for uploading action results to CAS after execution completes. If upload takes longer than this, the action fails with `DeadlineExceeded` and may be retried by the scheduler. Value in seconds. | +| `max_inflight_tasks` | integer (uint64) | — | 0 (infinite tasks) | Maximum number of inflight tasks this worker can cope with. | +| `timeout_handled_externally` | boolean | — | false (`NativeLink` fully handles timeouts) | If timeout is handled in `entrypoint` or another wrapper script. If set to true `NativeLink` will not honor the timeout the action requested and instead will always force kill the action after `max_action_timeout` has been reached. If this is set to false, the smaller value of the action's timeout and `max_action_timeout` will be used to which `NativeLink` will kill the action. | +| `entrypoint` | string | — | {Use the command from the job request} | The command to execute on every execution request. This will be parsed as a command + arguments (not shell). Example: "run.sh" and a job with command: "sleep 5" will result in a command like: "run.sh sleep 5". | +| `experimental_precondition_script` | string | — | — | An optional script to run before every action is processed on the worker. The value should be the full path to the script to execute and will pause all actions on the worker if it returns an exit code other than 0. If not set, then the worker will never pause and will continue to accept jobs according to the scheduler configuration. This is useful, for example, if the worker should not take any more actions until there is enough resource available on the machine to handle them. | +| `cas_fast_slow_store` | string | Yes | — | Underlying CAS store that the worker will use to download CAS artifacts. This store must be a `FastSlowStore`. The `fast` store must be a `FileSystemStore` because it will use hardlinks when building out the files instead of copying the files. The slow store must eventually resolve to the same store the scheduler/client uses to send job requests. | +| `upload_action_result` | [UploadActionResultConfig](#uploadactionresultconfig) | — | — | Configuration for uploading action results. | +| `work_directory` | string | Yes | — | The directory work jobs will be executed from. This directory will be fully managed by the worker service and will be purged on startup. This directory and the directory referenced in `local_filesystem_store_ref`'s `stores::FilesystemStore::content_path` must be on the same filesystem. Hardlinks will be used when placing files that are accessible to the jobs that are sourced from `local_filesystem_store_ref`'s `content_path`. | +| `platform_properties` | map of string to [WorkerProperty](#workerproperty) | Yes | — | Properties of this worker. This configuration will be sent to the scheduler and used to tell the scheduler to restrict what should be executed on this worker. | +| `additional_environment` | map of string to [EnvironmentSource](#environmentsource) | — | — | An optional mapping of environment names to set for the execution as well as those specified in the action itself. If set, will set each key as an environment variable before executing the job with the value of the environment variable being the value of the property of the action being executed of that name or the fixed value. | +| `directory_cache` | [DirectoryCacheConfig](#directorycacheconfig) | — | — | Optional directory cache configuration for improving performance by caching reconstructed input directories and using hardlinks instead of rebuilding them from CAS for every action. | +| `use_namespaces` | boolean | — | False | Whether to use namespaces to isolate the execution. This is only available on Linux. It is highly recommended as it avoids a number of issues with zombie processes and also provides additional hermeticity. If explicitly set to true and it is not supported the worker will exit with an error. | +| `use_mount_namespace` | boolean | — | False | Whether to use a mount namespace to isolate the worker root. This is only available on Linux and when `use_namespaces` is true. It is highly recommended provides additional hermeticity. If explicitly set to true and it is not supported or `use_namespaces` is not set to true the worker will exit with an error. | + +## SimpleSpec + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `supported_platform_properties` | map of string to [PropertyType](#propertytype) | — | — | A list of supported platform properties mapped to how these properties are used when the scheduler looks for worker nodes capable of running the task. | +| `retain_completed_for_s` | integer (uint32) | — | 60 seconds | The amount of time to retain completed actions for in case a `WaitExecution` is called after the action has completed. | +| `client_action_timeout_s` | integer (uint64) | — | 60 seconds | Mark operations as completed with error if no client has updated them within this duration. | +| `worker_timeout_s` | integer (uint64) | — | 5 seconds | Remove workers from pool once the worker has not responded in this amount of time in seconds. | +| `max_action_executing_timeout_s` | integer (uint64) | — | 0 (disabled) | Maximum time (seconds) an action can stay in Executing state without any worker update before being timed out and re-queued. This applies regardless of worker keepalive status, catching cases where a worker is alive (sending keepalives) but stuck on a specific action. Set to 0 to disable (relies only on `worker_timeout_s`). | +| `max_job_retries` | integer (uint) | — | 3 | If a job returns an internal error or times out this many times when attempting to run on a worker the scheduler will return the last error to the client. Jobs will be retried and this configuration is to help prevent one rogue job from infinitely retrying and taking up a lot of resources when the task itself is the one causing the server to go into a bad state. | +| `allocation_strategy` | [WorkerAllocationStrategy](#workerallocationstrategy) | — | `"least_recently_used"` | The strategy used to assign workers jobs. | +| `experimental_backend` | [ExperimentalSimpleSchedulerBackend](#experimentalsimpleschedulerbackend) | — | memory | The storage backend to use for the scheduler. | +| `worker_match_logging_interval_s` | integer (int64) | — | `10` | Every N seconds, do logging of worker matching e.g. "worker busy", "can't find any worker" Defaults to 10s. Can be set to `-1` to disable | + +## SchedulerGrpcSpec + +A scheduler that forwards requests to an upstream scheduler. This +is useful to use when doing some kind of local action cache or CAS away from +the main cluster of workers. In general, it's more efficient to point the +build at the main scheduler directly though. + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `endpoint` | [GrpcEndpoint](#grpcendpoint) | Yes | — | The upstream scheduler to forward requests to. | +| `retry` | [Retry](#retry) | — | — | Retry configuration to use when a network request fails. | +| `max_concurrent_requests` | integer (uint) | — | unlimited | Limit the number of simultaneous upstream requests to this many. A value of zero is treated as unlimited. If the limit is reached the request is queued. | +| `connections_per_endpoint` | integer (uint) | — | 1 | The number of connections to make to each specified endpoint to balance the load over multiple TCP connections. | + +## CacheLookupSpec + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `ac_store` | string | Yes | — | The reference to the action cache store used to return cached actions from rather than running them again. To prevent unintended issues, this store should probably be a `CompletenessCheckingSpec`. | +| `scheduler` | [SchedulerSpec](#schedulerspec) | Yes | — | The nested scheduler to use if cache lookup fails. | + +## PropertyModifierSpec + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `modifications` | array of [PropertyModification](#propertymodification) | Yes | — | A list of modifications to perform to incoming actions for the nested scheduler. These are performed in order and blindly, so removing a property that doesn't exist is fine and overwriting an existing property is also fine. If adding properties that do not exist in the nested scheduler is not supported and will likely cause unexpected behaviour. | +| `scheduler` | [SchedulerSpec](#schedulerspec) | Yes | — | The nested scheduler to use after modifying the properties. | + +## HistoricalResourceSpec + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `hints_file` | string | Yes | — | JSON file containing historical resource hints keyed by Bazel `RequestMetadata` `target_id` and/or `action_mnemonic`. | +| `refresh_interval_s` | integer (uint64) | — | 30 seconds | Reload interval for `hints_file`. Set to 0 to load once. | +| `cpu_property_name` | string | — | `cpu_count` | Platform property name used for CPU minimum values. | +| `memory_property_name` | string | — | `memory_kb` | Platform property name used for memory minimum values, expressed in KiB. | +| `scheduler` | [SchedulerSpec](#schedulerspec) | Yes | — | The nested scheduler to use after applying resource hints. | + +## ListenerConfig + +Plus exactly one of the following variants (the key selects the variant): + +### `http` + +Listener for HTTP/HTTPS/HTTP2 sockets. + +**Type:** [HttpListener](#httplistener) + +## ServicesConfig + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `cas` | array of [CasServiceConfig](#casserviceconfig) | — | — | The Content Addressable Storage (CAS) backend config. The key is the `instance_name` used in the protocol and the value is the underlying CAS store config. | +| `ac` | array of [ActionCacheServiceConfig](#actioncacheserviceconfig) | — | — | The Action Cache (AC) backend config. The key is the `instance_name` used in the protocol and the value is the underlying AC store config. | +| `capabilities` | array of [CapabilitiesServiceConfig](#capabilitiesserviceconfig) | — | — | Capabilities service is required in order to use most of the bazel protocol. This service is used to provide the supported features and versions of this bazel GRPC service. | +| `execution` | array of [ExecutionServiceConfig](#executionserviceconfig) | — | — | The remote execution service configuration. NOTE: This service is under development and is currently just a place holder. | +| `bytestream` | array of [ByteStreamServiceConfig](#bytestreamserviceconfig) | — | — | This is the service used to stream data to and from the CAS. Bazel's protocol strongly encourages users to use this streaming interface to interact with the CAS when the data is large. | +| `fetch` | array of [FetchServiceConfig](#fetchserviceconfig) | — | — | These two are collectively the Remote Asset protocol, but it's defined as two separate services | +| `push` | array of [PushServiceConfig](#pushserviceconfig) | — | — | | +| `worker_api` | [WorkerApiConfig](#workerapiconfig) | — | — | This is the service used for workers to connect and communicate through. NOTE: This service should be served on a different, non-public port. In other words, `worker_api` configuration should not have any other services that are served on the same port. Doing so is a security risk, as workers have a different permission set than a client that makes the remote execution/cache requests. | +| `experimental_bep` | [BepConfig](#bepconfig) | — | — | Experimental - Build Event Protocol (BEP) configuration. This is the service that will consume build events from the client and publish them to a store for processing by an external service. | +| `admin` | [AdminConfig](#adminconfig) | — | — | This is the service for any administrative tasks. It provides a REST API endpoint for administrative purposes. | +| `health` | [HealthConfig](#healthconfig) | — | — | This is the service for health status check. | + +## IdentityHeaderSpec + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `header_name` | string | — | "x-identity" | The name of the header to look for the identity in. | +| `required` | boolean | — | `false` | If the header is required to be set or fail the request. | + +## OriginEventsPublisherSpec + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `store` | string | Yes | — | The store to publish nativelink events to. The store name referenced in the `stores` map in the main config. | + +## ConfigDigestHashFunction + +| Value | Description | +| --- | --- | +| `"sha256"` | Use the sha256 hash function. [https://en.wikipedia.org/wiki/SHA-2](https://en.wikipedia.org/wiki/SHA-2) | +| `"blake3"` | Use the blake3 hash function. [https://en.wikipedia.org/wiki/BLAKE_(hash_function)](https://en.wikipedia.org/wiki/BLAKE_(hash_function)) | + +## EvictionPolicy + +Eviction policy always works on LRU (Least Recently Used). Any time an entry +is touched it updates the timestamp. Inserts and updates will execute the +eviction policy removing any expired entries and/or the oldest entries +until the store size becomes smaller than `max_bytes`. + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `max_bytes` | integer (uint) | — | 0 | Maximum number of bytes before eviction takes place. Zero means never evict based on size. | +| `evict_bytes` | integer (uint) | — | 0 | When eviction starts based on hitting `max_bytes`, continue until `max_bytes - evict_bytes` is met to create a low watermark. This stops operations from thrashing when the store is close to the limit. | +| `max_seconds` | integer (uint32) | — | 0 | Maximum number of seconds for an entry to live since it was last accessed before it is evicted. Zero means never evict based on time. | +| `max_count` | integer (uint64) | — | 0 | Maximum size of the store before an eviction takes place. Zero means never evict based on count. | + +## Retry + +Retry configuration. This configuration is exponential and each iteration +a jitter as a percentage is applied of the calculated delay. For example: +```haskell +Retry{ + max_retries: 7, + delay: 0.1, + jitter: 0.5, +} +``` +will result in: + +| Attempt | Delay | +| --- | --- | +| 1 | 0 ms | +| 2 | 75 to 125 ms | +| 3 | 150 to 250 ms | +| 4 | 300 to 500 ms | +| 5 | 600 ms to 1 s | +| 6 | 1.2 to 2 s | +| 7 | 2.4 to 4 s | +| 8 | 4.8 to 8 s | + +The total delay is additive, so this example produces 9.525 to 15.875 s of total delay for a single request. + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `max_retries` | integer (uint) | — | `0` | Maximum number of retries until retrying stops. Setting this to zero will always attempt 1 time, but not retry. | +| `delay` | number (float) | — | `0` | Delay in seconds for exponential back off. | +| `jitter` | number (float) | — | `0` | Amount of jitter to add as a percentage in decimal form. This will change the formula like: | +| `retry_on_errors` | array of [ErrorCode](#errorcode) | — | — | A list of error codes to retry on, if this is not set then the default error codes to retry on are used. These default codes are the most likely to be non-permanent: `Unknown`; `Cancelled`; `DeadlineExceeded`; `ResourceExhausted`; `Aborted`; `Internal`; `Unavailable`; `DataLoss` | + +## ExperimentalOntapS3Spec + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `endpoint` | string | Yes | — | | +| `vserver_name` | string | Yes | — | | +| `bucket` | string | Yes | — | | +| `root_certificates` | string | — | — | | +| `key_prefix` | string | — | — | If you wish to prefix the location in the bucket. If None, no prefix will be used. | +| `retry` | [Retry](#retry) | — | — | Retry configuration to use when a network request fails. | +| `consider_expired_after_s` | integer (uint32) | — | 0 | If the number of seconds since the `last_modified` time of the object is greater than this value, the object will not be considered "existing". This allows for external tools to delete objects that have not been uploaded in a long time. If a client receives a `NotFound` the client should re-upload the object. | +| `max_retry_buffer_per_request` | integer (uint) | — | 5MB | The maximum buffer size to retain in case of a retryable error during upload. Setting this to zero will disable upload buffering; this means that in the event of a failure during upload, the entire upload will be aborted and the client will likely receive an error. | +| `multipart_max_concurrent_uploads` | integer (uint) | — | 10 | Maximum number of concurrent `UploadPart` requests per `MultipartUpload`. | +| `insecure_allow_http` | boolean | — | false | Allow unencrypted HTTP connections. Only use this for local testing. | +| `disable_http2` | boolean | — | false | Disable HTTP/2 connections and only use HTTP/1.1. Default client configuration will have HTTP/1.1 and HTTP/2 enabled for connection schemes. HTTP/2 should be disabled if environments have poor support or performance related to HTTP/2. Safe to keep default unless underlying network environment, S3, or GCS API servers specify otherwise. | + +## CompressionAlgorithm + +Plus exactly one of the following variants (the key selects the variant): + +### `lz4` + +LZ4 compression algorithm is extremely fast for compression and +decompression, however does not perform very well in compression +ratio. In most cases build artifacts are highly compressible, however +lz4 is quite good at aborting early if the data is not deemed very +compressible. + +see: [https://lz4.github.io/lz4/](https://lz4.github.io/lz4/) + +**Type:** [Lz4Config](#lz4config) + +## StoreDirection + +| Value | Description | +| --- | --- | +| `"both"` | The store operates normally and all get and put operations are handled by it. | +| `"update"` | Update operations will cause persistence to this store, but Get operations will be ignored. This only makes sense on the fast store as the slow store will never get written to on Get anyway. | +| `"get"` | Get operations will cause persistence to this store, but Update operations will be ignored. | +| `"read_only"` | Operate as a read only store, only really makes sense if there's another way to write to it. | + +## ShardConfig + +Configuration for an individual shard of the store. + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `store` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | Store to shard the data to. | +| `weight` | integer (uint32) | — | 1 | The weight of the store. This is used to determine how much data should be sent to the store. The actual percentage is the sum of all the store's weights divided by the individual store's weight. | + +## GrpcEndpoint + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `address` | string | Yes | — | The endpoint address (i.e. grpc://example.com:443 or grpcs://example.com:443). | +| `tls_config` | [ClientTlsConfig](#clienttlsconfig) | — | — | The TLS configuration to use to connect to the endpoint (if grpcs). | +| `concurrency_limit` | integer (uint) | — | — | The maximum concurrency to allow on this endpoint. | +| `connect_timeout_s` | integer (uint64) | — | 30 seconds | Timeout for establishing a TCP connection to the endpoint (seconds). | +| `tcp_keepalive_s` | integer (uint64) | — | 30 seconds | TCP keepalive interval (seconds). Sends TCP keepalive probes at this interval to detect dead connections at the OS level. | +| `http2_keepalive_interval_s` | integer (uint64) | — | 30 seconds | HTTP/2 keepalive interval (seconds). Sends HTTP/2 PING frames at this interval to detect dead connections at the application level. | +| `http2_keepalive_timeout_s` | integer (uint64) | — | 20 seconds | HTTP/2 keepalive timeout (seconds). If a PING response is not received within this duration, the connection is considered dead. | + +## StoreType + +| Value | Description | +| --- | --- | +| `"cas"` | The store is content addressable storage. | +| `"ac"` | The store is an action cache. | + +## RedisMode + +| Value | Description | +| --- | --- | +| `"cluster"` | Use Redis Cluster. | +| `"sentinel"` | Use Redis Sentinel. | +| `"standard"` | Use a standalone Redis server. | + +## EndpointConfig + +Generic config for an endpoint and associated configs. + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `uri` | string | Yes | — | URI of the endpoint. | +| `timeout` | number (float) | — | 5 seconds | Timeout in seconds that a request should take. | +| `tls_config` | [ClientTlsConfig](#clienttlsconfig) | — | — | The TLS configuration to use to connect to the endpoint. | + +## UploadActionResultConfig + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `ac_store` | string | — | {No uploading is done} | Underlying AC store that the worker will use to publish execution results into. Objects placed in this store should be reachable from the scheduler/client-cas after they have finished updating. | +| `upload_ac_results_strategy` | [UploadCacheResultsStrategy](#uploadcacheresultsstrategy) | — | `SuccessOnly` | In which situations should the results be published to the `ac_store`, if set to `SuccessOnly` then only results with an exit code of 0 will be uploaded, if set to Everything all completed results will be uploaded. | +| `historical_results_store` | string | — | {CAS store of parent} | Store to upload historical results to. This should be a CAS store if set. | +| `upload_historical_results_strategy` | [UploadCacheResultsStrategy](#uploadcacheresultsstrategy) | — | `FailuresOnly` | In which situations should the results be published to the historical CAS. The historical CAS is where failures are published. These messages conform to the CAS key-value lookup format and are always a `HistoricalExecuteResponse` serialized message. | +| `success_message_template` | string | — | "" (no message) | Template to use for the `ExecuteResponse.message` property. This message is attached to the response before it is sent to the client. The following special variables are supported:; `digest_function`: Digest function used to calculate the action digest: `action_digest_hash`: Action digest hash: `action_digest_size`: Action digest size: `historical_results_hash`: `HistoricalExecuteResponse` digest hash: `historical_results_size`: `HistoricalExecuteResponse` digest size. | +| `failure_message_template` | string | — | "" (no message) | Same as `success_message_template` but for failure case. | + +## WorkerProperty + +Plus exactly one of the following variants (the key selects the variant): + +### `values` + +List of static values. +Note: Generally there should only ever be 1 value, but if the platform +property key is `PropertyType::Priority` it may have more than one value. + +**Type:** array of string + +### `query_cmd` + +A dynamic configuration. The string will be executed as a command +(not shell) and will be split by "\n" (new line character). + +**Type:** string + +## EnvironmentSource + +One of: + +- `property`: string — The name of the platform property in the action to get the value from. +- `value`: string — The raw value to set. +- `"from_environment"` — Take the value from the local environment corresponding to the name key +- `"timeout_millis"` — The max amount of time in milliseconds the command is allowed to run (requested by the client). +- `"side_channel_file"` — A special file path will be provided that can be used to communicate with the parent process about out-of-band information. This file will be read after the command has finished executing. Based on the contents of the file, the behavior of the result may be modified. +- `"action_directory"` — A "root" directory for the action. This directory can be used to store temporary files that are not needed after the action has completed. This directory will be purged after the action has completed. + +## DirectoryCacheConfig + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `max_entries` | integer (uint) | — | 1000 | Maximum number of cached directories. | +| `max_size_bytes` | integer (uint64) | — | 10737418240 (10 GB) | Maximum total size in bytes for all cached directories (0 = unlimited). | +| `cache_root` | string | — | `{work_directory}/../directory_cache` | Base directory for cache storage. This directory will be managed by the worker and should be on the same filesystem as `work_directory`. | + +## PropertyType + +When the scheduler matches tasks to workers that are capable of running +the task, this value will be used to determine how the property is treated. + +One of: + +- string +- `"minimum"` — Requires the platform property to be a u64 and when the scheduler looks for appropriate worker nodes that are capable of executing the task, the task will not run on a node that has less than this value. +- `"exact"` — Requires the platform property to be a string and when the scheduler looks for appropriate worker nodes that are capable of executing the task, the task will not run on a node that does not have this property set to the value with exact string match. +- `"priority"` — Does not restrict on this value and instead will be passed to the worker as an informational piece. In the future this will be used by the scheduler and worker to cause the scheduler to prefer certain workers over others, but not restrict them based on these values. + +## WorkerAllocationStrategy + +When a worker is being searched for to run a job, this will be used +on how to choose which worker should run the job when multiple +workers are able to run the task. + +| Value | Description | +| --- | --- | +| `"least_recently_used"` | Prefer workers that have been least recently used to run a job. | +| `"most_recently_used"` | Prefer workers that have been most recently used to run a job. | + +## ExperimentalSimpleSchedulerBackend + +One of: + +- `"memory"` — Use an in-memory store for the scheduler. +- `redis`: [ExperimentalRedisSchedulerBackend](#experimentalredisschedulerbackend) — Use a Redis store for the scheduler. + +## SchedulerSpec + +Plus exactly one of the following variants (the key selects the variant): + +### `simple` + +**Type:** [SimpleSpec](#simplespec) + +### `grpc` + +**Type:** [SchedulerGrpcSpec](#schedulergrpcspec) + +### `cache_lookup` + +**Type:** [CacheLookupSpec](#cachelookupspec) + +### `property_modifier` + +**Type:** [PropertyModifierSpec](#propertymodifierspec) + +### `historical_resource` + +**Type:** [HistoricalResourceSpec](#historicalresourcespec) + +## PropertyModification + +Plus exactly one of the following variants (the key selects the variant): + +### `add` + +Add a property to the action properties. + +**Type:** [PlatformPropertyAddition](#platformpropertyaddition) + +### `remove` + +Remove a named property from the action. + +**Type:** string + +### `replace` + +If a property is found, then replace it with another one. + +**Type:** [PlatformPropertyReplacement](#platformpropertyreplacement) + +## HttpListener + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `socket_address` | string | Yes | — | Address to listen on. Example: `127.0.0.1:8080` or `:8080` to listen to all IPs. | +| `freebind` | boolean | — | false | Allow binding `socket_address` before it is assigned locally. | +| `compression` | [HttpCompressionConfig](#httpcompressionconfig) | — | — | Data transport compression configuration to use for this service. | +| `advanced_http` | [HttpServerConfig](#httpserverconfig) | — | — | Advanced HTTP server configuration. | +| `max_decoding_message_size` | integer (uint) | — | 4 MiB | Maximum number of bytes to decode on each grpc stream chunk. | +| `tls` | [TlsConfig](#tlsconfig) | — | — | TLS configuration for this server. If not set, the server will not use TLS. | + +## CasServiceConfig + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `instance_name` | string | — | `""` | Used when the config references `instance_name` in the protocol. | +| `cas_store` | string | Yes | — | The store name referenced in the `stores` map in the main config. This store name referenced here may be reused multiple times. | + +## ActionCacheServiceConfig + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `instance_name` | string | — | `""` | Used when the config references `instance_name` in the protocol. | +| `ac_store` | string | Yes | — | The store name referenced in the `stores` map in the main config. This store name referenced here may be reused multiple times. | +| `read_only` | boolean | — | `false` | Whether the Action Cache store may be written to, this if set to false it is only possible to read from the Action Cache. | + +## CapabilitiesServiceConfig + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `instance_name` | string | — | `""` | Used when the config references `instance_name` in the protocol. | +| `remote_execution` | [CapabilitiesRemoteExecutionConfig](#capabilitiesremoteexecutionconfig) | — | — | Configuration for remote execution capabilities. If not set the capabilities service will inform the client that remote execution is not supported. | + +## ExecutionServiceConfig + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `instance_name` | string | — | `""` | Used when the config references `instance_name` in the protocol. | +| `cas_store` | string | Yes | — | The store name referenced in the `stores` map in the main config. This store name referenced here may be reused multiple times. This value must be a CAS store reference. | +| `scheduler` | string | Yes | — | The scheduler name referenced in the `schedulers` map in the main config. | + +## ByteStreamServiceConfig + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `instance_name` | string | — | `""` | Used when the config references `instance_name` in the protocol. | +| `cas_store` | string | Yes | — | Name of the store in the "stores" configuration. | +| `max_bytes_per_stream` | integer (uint) | — | 64KiB | Max number of bytes to send on each grpc stream chunk. According to [https://github.com/grpc/grpc.github.io/issues/371](https://github.com/grpc/grpc.github.io/issues/371) 16KiB - 64KiB is optimal. | +| `persist_stream_on_disconnect_timeout` | integer (uint) | — | 10 seconds | In the event a client disconnects while uploading a blob, we will hold the internal stream open for this many seconds before closing it. This allows clients that disconnect to reconnect and continue uploading the same blob. | + +## FetchServiceConfig + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `instance_name` | string | — | `""` | Used when the config references `instance_name` in the protocol. | +| `fetch_store` | string | Yes | — | The store name referenced in the `stores` map in the main config. This store name referenced here may be reused multiple times. | + +## PushServiceConfig + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `instance_name` | string | — | `""` | Used when the config references `instance_name` in the protocol. | +| `push_store` | string | Yes | — | The store name referenced in the `stores` map in the main config. This store name referenced here may be reused multiple times. | +| `read_only` | boolean | — | `false` | Whether the Action Cache store may be written to, this if set to false it is only possible to read from the Action Cache. | + +## WorkerApiConfig + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `scheduler` | string | Yes | — | The scheduler name referenced in the `schedulers` map in the main config. | + +## BepConfig + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `store` | string | Yes | — | The store to publish build events to. The store name referenced in the `stores` map in the main config. | + +## AdminConfig + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `path` | string | — | "/admin" | Path to register the admin API. If path is "/admin", and your domain is "example.com", you can reach the endpoint with: [http://example.com/admin](http://example.com/admin). | + +## HealthConfig + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `path` | string | — | "/status" | Path to register the health status check. If path is "/status", and your domain is "example.com", you can reach the endpoint with: [http://example.com/status](http://example.com/status). | +| `timeout_seconds` | integer (uint64) | — | 5s | Timeout on health checks. | + +## ErrorCode + +The possible error codes that might occur on an upstream request. + +| Value | Description | +| --- | --- | +| `"Cancelled"` | | +| `"Unknown"` | | +| `"InvalidArgument"` | | +| `"DeadlineExceeded"` | | +| `"NotFound"` | | +| `"AlreadyExists"` | | +| `"PermissionDenied"` | | +| `"ResourceExhausted"` | | +| `"FailedPrecondition"` | | +| `"Aborted"` | | +| `"OutOfRange"` | | +| `"Unimplemented"` | | +| `"Internal"` | | +| `"Unavailable"` | | +| `"DataLoss"` | | +| `"Unauthenticated"` | | + +## Lz4Config + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `block_size` | integer (uint32) | — | 65536 (64k) | Size of the blocks to compress. Higher values require more ram, but might yield slightly better compression ratios. | +| `max_decode_block_size` | integer (uint32) | — | value in `block_size` | Maximum size allowed to attempt to deserialize data into. This is needed because the `block_size` is embedded into the data so if there was a bad actor, they could upload an extremely large `block_size`'ed entry and we'd allocate a large amount of memory when retrieving the data. To prevent this from happening, we allow you to specify the maximum that we'll attempt to deserialize. | + +## ClientTlsConfig + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `ca_file` | string | — | — | Path to the certificate authority to use to validate the remote. | +| `cert_file` | string | — | — | Path to the certificate file for client authentication. | +| `key_file` | string | — | — | Path to the private key file for client authentication. | +| `use_native_roots` | boolean | — | false | If set the client will use the native roots for TLS connections. | + +## UploadCacheResultsStrategy + +| Value | Description | +| --- | --- | +| `"success_only"` | Only upload action results with an exit code of 0. | +| `"never"` | Don't upload any action results. | +| `"everything"` | Upload all action results that complete. | +| `"failures_only"` | Only upload action results that fail. | + +## ExperimentalRedisSchedulerBackend + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `redis_store` | string | Yes | — | A reference to the Redis store to use for the scheduler. Note: This MUST resolve to a `RedisSpec`. | + +## PlatformPropertyAddition + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `name` | string | Yes | — | The name of the property to add. | +| `value` | string | Yes | — | The value to assign to the property. | + +## PlatformPropertyReplacement + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `name` | string | Yes | — | The name of the property to replace. | +| `value` | string | — | — | The value to match against, if unset then any instance matches. | +| `new_name` | string | Yes | — | The new name of the property. | +| `new_value` | string | — | — | The value to assign to the property, if unset will remain the same. | + +## HttpCompressionConfig + +Note: Compressing data in the cloud rarely has a benefit, since most +cloud providers have very high bandwidth backplanes. However, for +clients not inside the data center, it might be a good idea to +compress data to and from the cloud. This will however come at a high +CPU and performance cost. If you are making remote execution share the +same CAS/AC servers as client's remote cache, you can create multiple +services with different compression settings that are served on +different ports. Then configure the non-cloud clients to use one port +and cloud-clients to use another. + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `send_compression_algorithm` | [HttpCompressionAlgorithm](#httpcompressionalgorithm) | — | `HttpCompressionAlgorithm::None` | The compression algorithm that the server will use when sending responses to clients. Enabling this will likely save a lot of data transfer, but will consume a lot of CPU and add a lot of latency. see: [https://github.com/tracemachina/nativelink/issues/109](https://github.com/tracemachina/nativelink/issues/109) | +| `accepted_compression_algorithms` | array of [HttpCompressionAlgorithm](#httpcompressionalgorithm) | Yes | {no supported compression} | The compression algorithm that the server will accept from clients. The server will broadcast the supported compression algorithms to clients and the client will choose which compression algorithm to use. Enabling this will likely save a lot of data transfer, but will consume a lot of CPU and add a lot of latency. see: [https://github.com/tracemachina/nativelink/issues/109](https://github.com/tracemachina/nativelink/issues/109) | + +## HttpServerConfig + +Advanced HTTP configuration. These generally should not be set. +For documentation on these settings, see the hyper documentation: +See: [hyper HTTP server docs](https://docs.rs/hyper/latest/hyper/server/conn/struct.Http.html) + +Note: All of these default to the default values from hyper unless otherwise +specified. + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `http2_keep_alive_interval` | integer (uint32) | — | — | Interval to send keep-alive pings via HTTP2. Note: This is in seconds. | +| `experimental_http2_max_pending_accept_reset_streams` | integer (uint32) | — | — | | +| `experimental_http2_initial_stream_window_size` | integer (uint32) | — | — | | +| `experimental_http2_initial_connection_window_size` | integer (uint32) | — | — | | +| `experimental_http2_adaptive_window` | boolean | — | — | | +| `experimental_http2_max_frame_size` | integer (uint32) | — | — | | +| `experimental_http2_max_concurrent_streams` | integer (uint32) | — | — | | +| `experimental_http2_keep_alive_timeout` | integer (uint32) | — | — | Note: This is in seconds. | +| `experimental_http2_max_send_buf_size` | integer (uint32) | — | — | | +| `experimental_http2_enable_connect_protocol` | boolean | — | — | | +| `experimental_http2_max_header_list_size` | integer (uint32) | — | — | | + +## TlsConfig + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `cert_file` | string | Yes | — | Path to the certificate file. | +| `key_file` | string | Yes | — | Path to the private key file. | +| `client_ca_file` | string | — | — | Path to the certificate authority for mTLS, if client authentication is required for this endpoint. | +| `client_crl_file` | string | — | — | Path to the certificate revocation list for mTLS, if client authentication is required for this endpoint. | + +## CapabilitiesRemoteExecutionConfig + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `scheduler` | string | Yes | — | Scheduler used to configure the capabilities of remote execution. | + +## HttpCompressionAlgorithm + +| Value | Description | +| --- | --- | +| `"none"` | No compression. | +| `"gzip"` | Zlib compression. | + +## Reading the source + +If anything here disagrees with the binary, the source wins: + +- [`stores.rs`](https://github.com/TraceMachina/nativelink/tree/v1.5.2/nativelink-config/src/stores.rs) +- [`cas_server.rs`](https://github.com/TraceMachina/nativelink/tree/v1.5.2/nativelink-config/src/cas_server.rs) +- [`schedulers.rs`](https://github.com/TraceMachina/nativelink/tree/v1.5.2/nativelink-config/src/schedulers.rs) diff --git a/web/apps/docs/content/docs/reference/nativelink-config/v1.6.0.mdx b/web/apps/docs/content/docs/reference/nativelink-config/v1.6.0.mdx new file mode 100644 index 000000000..74644a0e1 --- /dev/null +++ b/web/apps/docs/content/docs/reference/nativelink-config/v1.6.0.mdx @@ -0,0 +1,1511 @@ +--- +title: Configuration reference +description: Every knob in the NativeLink JSON5 configuration — types, defaults, and links to source, autogenerated from the Rust config crate. +full: true +--- + +{/* AUTOGENERATED — do not edit by hand. + Source: nativelink-config @ v1.6.0 (05c5fac5) + Regenerate from web/: bun --filter @nativelink/docs gen:config-reference */} + + + +This is the canonical NativeLink configuration reference for **v1.6.0**. +It is autogenerated from the Rust config crate +([`nativelink-config/src`](https://github.com/TraceMachina/nativelink/tree/v1.6.0/nativelink-config/src)) via the `build-schema` binary, so +it can never drift from what the binary actually deserializes. + +## Top-level fields + +The root object (`CasConfig`) accepts the following fields: + +| Field | Type | Required | Description | +| --- | --- | --- | --- | +| `stores` | array of [NamedStoreConfig](#namedstoreconfig) | Yes | List of stores available to use in this config. The keys can be used in other configs when needing to reference a store. | +| `workers` | array of [WorkerConfig](#workerconfig) | — | Worker configurations used to execute jobs. | +| `schedulers` | array of [NamedSchedulerConfig](#namedschedulerconfig) | — | List of schedulers available to use in this config. The keys can be used in other configs when needing to reference a scheduler. | +| `servers` | array of [ServerConfig](#serverconfig) | Yes | Servers to setup for this process. | +| `experimental_origin_events` | [OriginEventsSpec](#origineventsspec) | — | Experimental - Origin events configuration. This is the service that will collect and publish nativelink events to a store for processing by an external service. | +| `global` | [GlobalConfig](#globalconfig) | — | Any global configurations that apply to all modules live here. | + +## Configuration types + +Every type reachable from the root configuration, in reading order. + +## NamedStoreConfig + +**Common fields** + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `name` | string | Yes | — | | + +Plus exactly one of the following variants (the key selects the variant): + +### `cache_metrics` + +Cache metrics store wraps another store and emits low-cardinality +OpenTelemetry cache operation metrics for the wrapped store. + +This wrapper is opt-in. Stores that are not explicitly wrapped by +`cache_metrics` are constructed exactly as they are without this +wrapper and do not pay its hot-path timing or recording cost. + +**Example JSON5 config:** +```json5 +"cache_metrics": { + "cache_type": "cas", + "backend": { + "filesystem": { + "content_path": "~/.cache/nativelink/content_path-cas", + "temp_path": "~/.cache/nativelink/tmp_path-cas" + } + } +} +``` + +**Type:** [CacheMetricsSpec](#cachemetricsspec) + +### `memory` + +Memory store will store all data in a hash map in memory. + +**Example JSON5 config:** +```json5 +"memory": { + "eviction_policy": { + "max_bytes": "10mb", + } +} +``` + +**Type:** [MemorySpec](#memoryspec) + +### `experimental_cloud_object_store` + +A generic blob store that will store files on the cloud +provider. This configuration will never delete files, so you are +responsible for purging old files in other ways. +It supports the following backends: + +1. **Amazon S3:** + S3 store will use Amazon's S3 service as a backend to store + the files. This configuration can be used to share files + across multiple instances. Uses system certificates for TLS + verification via `rustls-platform-verifier`. + + **Example JSON5 config:** + ```json5 + "experimental_cloud_object_store": { + "provider": "aws", + "region": "eu-north-1", + "bucket": "crossplane-bucket-af79aeca9", + "key_prefix": "test-prefix-index/", + "retry": { + "max_retries": 6, + "delay": 0.3, + "jitter": 0.5 + }, + "multipart_max_concurrent_uploads": 10 + } + ``` + +2. **Google Cloud Storage:** + GCS store uses Google's GCS service as a backend to store + the files. This configuration can be used to share files + across multiple instances. + + **Example JSON5 config:** + ```json5 + "experimental_cloud_object_store": { + "provider": "gcs", + "bucket": "test-bucket", + "key_prefix": "test-prefix-index/", + "retry": { + "max_retries": 6, + "delay": 0.3, + "jitter": 0.5 + }, + "multipart_max_concurrent_uploads": 10 + } + ``` + +3. **Azure Blob Store:** + Azure Blob store will use Microsoft's Azure Blob service as a + backend to store the files. This configuration can be used to + share files across multiple instances. + + **Example JSON5 config:** + ```json5 + "experimental_cloud_object_store": { + "provider": "azure", + "account_name": "cloudshell1393657559", + "container": "simple-test-container", + "key_prefix": "folder/", + "retry": { + "max_retries": 6, + "delay": 0.3, + "jitter": 0.5 + }, + "multipart_max_concurrent_uploads": 10 + } + ``` + +4. **NetApp ONTAP S3:** + NetApp ONTAP S3 store will use ONTAP's S3-compatible storage as a backend + to store files. This store is specifically configured for ONTAP's S3 requirements + including custom TLS configuration, credentials management, and proper vserver + configuration. + + This store uses AWS environment variables for credentials: + - `AWS_ACCESS_KEY_ID` + - `AWS_SECRET_ACCESS_KEY` + - `AWS_DEFAULT_REGION` + + **Example JSON5 config:** + ```json5 + "experimental_cloud_object_store": { + "provider": "ontap", + "endpoint": "https://ontap-s3-endpoint:443", + "vserver_name": "your-vserver", + "bucket": "your-bucket", + "root_certificates": "/path/to/certs.pem", // Optional + "key_prefix": "test-prefix/", // Optional + "retry": { + "max_retries": 6, + "delay": 0.3, + "jitter": 0.5 + }, + "multipart_max_concurrent_uploads": 10 + } + ``` + +5. **Cloudflare R2:** + R2 store uses Cloudflare's R2 service as a backend. R2 speaks the + S3 API, so this is a thin wrapper that derives the account-scoped + endpoint (`https://{account_id}.r2.cloudflarestorage.com`) for you. + + **Example JSON5 config:** + ```json5 + "experimental_cloud_object_store": { + "provider": "r2", + "account_id": "a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4", + "bucket": "nativelink-cas", + "key_prefix": "test-prefix/", + "retry": { + "max_retries": 6, + "delay": 0.3, + "jitter": 0.5 + }, + "multipart_max_concurrent_uploads": 10 + } + ``` + +6. **Oracle Cloud Infrastructure (OCI) Object Storage:** + OCI store uses Oracle Cloud Infrastructure's S3-compatible Object + Storage API. The path-style endpoint is derived from your Object + Storage `namespace` and `region` as + `https://{namespace}.compat.objectstorage.{region}.oci.customer-oci.com`. + Authenticate with a Customer Secret Key (Access Key/Secret Key pair + created under User Settings -> Customer secret keys in the OCI + console); the secret cannot be retrieved after generation, so read + it from an env var via shellexpand. + + **Example JSON5 config:** + ```json5 + "experimental_cloud_object_store": { + "provider": "oci", + "namespace": "your-object-storage-namespace", + "region": "us-phoenix-1", + "bucket": "nativelink-cas", + "access_key_id": "oci_access_key_id", + "secret_access_key": "oci_secret_access_key", + "key_prefix": "test-prefix/", + "retry": { + "max_retries": 6, + "delay": 0.3, + "jitter": 0.5 + } + } + ``` + +**Type:** [ExperimentalCloudObjectSpec](#experimentalcloudobjectspec) + +### `ontap_s3_existence_cache` + +ONTAP S3 Existence Cache provides a caching layer on top of the ONTAP S3 store +to optimize repeated existence checks. It maintains an in-memory cache of object +digests and periodically syncs this cache to disk for persistence. + +The cache helps reduce latency for repeated calls to check object existence, +while still ensuring eventual consistency with the underlying ONTAP S3 store. + +Example JSON5 config: +```json5 +"ontap_s3_existence_cache": { + "index_path": "/path/to/cache/index.json", + "sync_interval_seconds": 300, + "backend": { + "endpoint": "https://ontap-s3-endpoint:443", + "vserver_name": "your-vserver", + "bucket": "your-bucket", + "key_prefix": "test-prefix/" + } +} +``` + +**Type:** [OntapS3ExistenceCacheSpec](#ontaps3existencecachespec) + +### `verify` + +Verify store is used to apply verifications to an underlying +store implementation. It is strongly encouraged to validate +as much data as you can before accepting data from a client, +failing to do so may cause the data in the store to be +populated with invalid data causing all kinds of problems. + +The suggested configuration is to have the CAS validate the +hash and size and the AC validate nothing. + +**Example JSON5 config:** +```json5 +"verify": { + "backend": { + "memory": { + "eviction_policy": { + "max_bytes": "500mb" + } + }, + }, + "verify_size": true, + "verify_hash": true +} +``` + +**Type:** [VerifySpec](#verifyspec) + +### `completeness_checking` + +Completeness checking store verifies if the +output files & folders exist in the CAS before forwarding +the request to the underlying store. +Note: This store should only be used on AC stores. + +**Example JSON5 config:** +```json5 +"completeness_checking": { + "backend": { + "filesystem": { + "content_path": "~/.cache/nativelink/content_path-ac", + "temp_path": "~/.cache/nativelink/tmp_path-ac", + "eviction_policy": { + "max_bytes": "500mb", + } + } + }, + "cas_store": { + "ref_store": { + "name": "CAS_MAIN_STORE" + } + } +} +``` + +**Type:** [CompletenessCheckingSpec](#completenesscheckingspec) + +### `compression` + +A compression store that will compress the data inbound and +outbound. There will be a non-trivial cost to compress and +decompress the data, but in many cases if the final store is +a store that requires network transport and/or storage space +is a concern it is often faster and more efficient to use this +store before those stores. + +**Example JSON5 config:** +```json5 +"compression": { + "compression_algorithm": { + "lz4": {} + }, + "backend": { + "filesystem": { + "content_path": "/tmp/nativelink/data/content_path-cas", + "temp_path": "/tmp/nativelink/data/tmp_path-cas", + "eviction_policy": { + "max_bytes": "2gb", + } + } + } +} +``` + +**Type:** [CompressionSpec](#compressionspec) + +### `dedup` + +A dedup store will take the inputs and run a rolling hash +algorithm on them to slice the input into smaller parts then +run a sha256 algorithm on the slice and if the object doesn't +already exist, upload the slice to the `content_store` using +a new digest of just the slice. Once all parts exist, an +Action-Cache-like digest will be built and uploaded to the +`index_store` which will contain a reference to each +chunk/digest of the uploaded file. Downloading a request will +first grab the index from the `index_store`, and forward the +download content of each chunk as if it were one file. + +This store is exceptionally good when the following conditions +are met: +* Content is mostly the same (inserts, updates, deletes are ok) +* Content is not compressed or encrypted +* Uploading or downloading from `content_store` is the bottleneck. + +Note: This store pairs well when used with `CompressionSpec` as +the `content_store`, but never put `DedupSpec` as the backend of +`CompressionSpec` as it will negate all the gains. + +Note: When running `.has()` on this store, it will only check +to see if the entry exists in the `index_store` and not check +if the individual chunks exist in the `content_store`. + +**Example JSON5 config:** +```json5 +"dedup": { + "index_store": { + "memory": { + "eviction_policy": { + "max_bytes": "1GB", + } + } + }, + "content_store": { + "compression": { + "compression_algorithm": { + "lz4": {} + }, + "backend": { + "fast_slow": { + "fast": { + "memory": { + "eviction_policy": { + "max_bytes": "500MB", + } + } + }, + "slow": { + "filesystem": { + "content_path": "/tmp/nativelink/data/content_path-content", + "temp_path": "/tmp/nativelink/data/tmp_path-content", + "eviction_policy": { + "max_bytes": "2gb" + } + } + } + } + } + } + } +} +``` + +**Type:** [DedupSpec](#dedupspec) + +### `existence_cache` + +Existence store will wrap around another store and cache calls +to has so that subsequent `has_with_results` calls will be +faster. This is useful for cases when you have a store that +is slow to respond to has calls. +Note: This store should only be used on CAS stores. + +**Example JSON5 config:** +```json5 +"existence_cache": { + "backend": { + "memory": { + "eviction_policy": { + "max_bytes": "500mb", + } + } + }, + // Note this is the existence store policy, not the backend policy + "eviction_policy": { + "max_seconds": 100, + } +} +``` + +**Type:** [ExistenceCacheSpec](#existencecachespec) + +### `fast_slow` + +`FastSlow` store will first try to fetch the data from the `fast` +store and then if it does not exist try the `slow` store. +When the object does exist in the `slow` store, it will copy +the data to the `fast` store while returning the data. +This store should be thought of as a store that "buffers" +the data to the `fast` store. +On uploads it will mirror data to both `fast` and `slow` stores. + +WARNING: If you need data to always exist in the `slow` store +for something like remote execution, be careful because this +store will never check to see if the objects exist in the +`slow` store if it exists in the `fast` store (i.e., it assumes +that if an object exists in the `fast` store it will exist in +the `slow` store). + +***Example JSON5 config:*** +```json5 +"fast_slow": { + "fast": { + "filesystem": { + "content_path": "/tmp/nativelink/data/content_path-index", + "temp_path": "/tmp/nativelink/data/tmp_path-index", + "eviction_policy": { + "max_bytes": "500mb", + } + } + }, + "slow": { + "filesystem": { + "content_path": "/tmp/nativelink/data/content_path-index", + "temp_path": "/tmp/nativelink/data/tmp_path-index", + "eviction_policy": { + "max_bytes": "500mb", + } + } + } +} +``` + +**Type:** [FastSlowSpec](#fastslowspec) + +### `shard` + +Shards the data to multiple stores. This is useful for cases +when you want to distribute the load across multiple stores. +The digest hash is used to determine which store to send the +data to. + +**Example JSON5 config:** +```json5 +"shard": { + "stores": [ + { + "store": { + "memory": { + "eviction_policy": { + "max_bytes": "10mb" + }, + }, + }, + "weight": 1 + }] +} +``` + +**Type:** [ShardSpec](#shardspec) + +### `filesystem` + +Stores the data on the filesystem. This store is designed for +local persistent storage. Restarts of this program should restore +the previous state, meaning anything uploaded will be persistent +as long as the filesystem integrity holds. + +**Example JSON5 config:** +```json5 +"filesystem": { + "content_path": "/tmp/nativelink/data-worker-test/content_path-cas", + "temp_path": "/tmp/nativelink/data-worker-test/tmp_path-cas", + "eviction_policy": { + "max_bytes": "10gb", + } +} +``` + +**Type:** [FilesystemSpec](#filesystemspec) + +### `ref_store` + +Store used to reference a store in the root store manager. +This is useful for cases when you want to share a store in different +nested stores. Example, you may want to share the same memory store +used for the action cache, but use a `FastSlowSpec` and have the fast +store also share the memory store for efficiency. + +**Example JSON5 config:** +```json5 +"ref_store": { + "name": "FS_CONTENT_STORE" +} +``` + +**Type:** [RefSpec](#refspec) + +### `size_partitioning` + +Uses the size field of the digest to separate which store to send the +data. This is useful for cases when you'd like to put small objects +in one store and large objects in another store. This should only be +used if the size field is the real size of the content, in other +words, don't use on AC (Action Cache) stores. Any store where you can +safely use `VerifySpec.verify_size = true`, this store should be safe +to use (i.e., CAS stores). + +**Example JSON5 config:** +```json5 +"size_partitioning": { + "size": "128mib", + "lower_store": { + "memory": { + "eviction_policy": { + "max_bytes": "${NATIVELINK_CAS_MEMORY_CONTENT_LIMIT:-100mb}" + } + } + }, + "upper_store": { + /// This store discards data larger than 128mib. + "noop": {} + } +} +``` + +**Type:** [SizePartitioningSpec](#sizepartitioningspec) + +### `grpc` + +This store will pass-through calls to another GRPC store. This store +is not designed to be used as a sub-store of another store, but it +does satisfy the interface and will likely work. + +One major GOTCHA is that some stores use a special function on this +store to get the size of the underlying object, which is only reliable +when this store is serving the a CAS store, not an AC store. If using +this store directly without being a child of any store there are no +side effects and is the most efficient way to use it. + +**Example JSON5 config:** +```json5 +"grpc": { + "instance_name": "main", + "endpoints": [ + {"address": "grpc://${CAS_ENDPOINT:-127.0.0.1}:50051"} + ], + "connections_per_endpoint": "5", + "rpc_timeout_s": "5m", + "store_type": "ac", + // Static headers attached to every outgoing request to the upstream + // remote cache. Useful for fixed service-account credentials. + "headers": { + "authorization": "Bearer my-static-token" + }, + // Header names to copy from the inbound client request and forward to + // the upstream remote cache. Use this to pass through dynamic + // credentials such as a JWT sent by the build client. + "forward_headers": ["authorization", "x-custom-token"] +} +``` + +**Type:** [GrpcSpec](#grpcspec) + +### `redis_store` + +Stores data in any stores compatible with Redis APIs. + +Pairs well with `SizePartitioning` and/or `FastSlow` stores. +Ideal for accepting small object sizes as most Redis store +services have a max file upload of between 256Mb-512Mb. + +**Example JSON5 config:** +```json5 +"redis_store": { + "addresses": [ + "redis://127.0.0.1:6379/", + ], + "max_client_permits": 1000, +} +``` + +**Type:** [RedisSpec](#redisspec) + +### `noop` + +Noop store is a store that sends streams into the void and all data +retrieval will return 404 (`NotFound`). This can be useful for cases +where you may need to partition your data and part of your data needs +to be discarded. + +**Example JSON5 config:** +```json5 +"noop": {} +``` + +**Type:** [NoopSpec](#noopspec) + +### `experimental_mongo` + +Experimental `MongoDB` store implementation. + +This store uses `MongoDB` as a backend for storing data. It supports +both CAS (Content Addressable Storage) and scheduler data with +optional change streams for real-time updates. + +**Example JSON5 config:** +```json5 +"experimental_mongo": { + "connection_string": "mongodb://localhost:27017", + "database": "nativelink", + "cas_collection": "cas", + "key_prefix": "cas:", + "read_chunk_size": 65536, + "max_concurrent_uploads": 10, + "enable_change_streams": false, + "max_requests": "100" +} +``` + +**Type:** [ExperimentalMongoSpec](#experimentalmongospec) + +## WorkerConfig + +Plus exactly one of the following variants (the key selects the variant): + +### `local` + +A worker type that executes jobs locally on this machine. + +**Type:** [LocalWorkerConfig](#localworkerconfig) + +## NamedSchedulerConfig + +**Common fields** + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `name` | string | Yes | — | | + +Plus exactly one of the following variants (the key selects the variant): + +### `simple` + +**Type:** [SimpleSpec](#simplespec) + +### `grpc` + +**Type:** [SchedulerGrpcSpec](#schedulergrpcspec) + +### `cache_lookup` + +**Type:** [CacheLookupSpec](#cachelookupspec) + +### `property_modifier` + +**Type:** [PropertyModifierSpec](#propertymodifierspec) + +### `historical_resource` + +**Type:** [HistoricalResourceSpec](#historicalresourcespec) + +## ServerConfig + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `name` | string | — | {index of server in config} | Name of the server. This is used to help identify the service for telemetry and logs. | +| `listener` | [ListenerConfig](#listenerconfig) | Yes | — | Configuration | +| `services` | [ServicesConfig](#servicesconfig) | — | — | Services to attach to server. | +| `experimental_identity_header` | [IdentityHeaderSpec](#identityheaderspec) | — | {see `IdentityHeaderSpec`} | The config related to identifying the client. | + +## OriginEventsSpec + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `publisher` | [OriginEventsPublisherSpec](#origineventspublisherspec) | Yes | — | The publisher configuration for origin events. | +| `max_event_queue_size` | integer (uint) | — | 65536 (zero defaults to this) | The maximum number of events to queue before applying back pressure. IMPORTANT: Backpressure causes all clients to slow down significantly. Zero is default. | + +## GlobalConfig + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `max_open_files` | integer (uint) | Yes | 24576 (= 24 * 1024) | Maximum number of open files that can be opened at one time. This value is not strictly enforced, it is a best effort. Some internal libraries open files or read metadata from a files which do not obey this limit, however the vast majority of cases will have this limit be honored. This value must be larger than `ulimit -n` to have any effect. Any network open file descriptors is not counted in this limit, but is counted in the kernel limit. It is a good idea to set a very large `ulimit -n`. Note: This value must be greater than 10. | +| `default_digest_hash_function` | [ConfigDigestHashFunction](#configdigesthashfunction) | — | `ConfigDigestHashFunction::sha256` | Default hash function to use while uploading blobs to the CAS when not set by client. | +| `default_digest_size_health_check` | integer (uint) | — | 1024*1024 (1MiB) | Default digest size to use for health check when running diagnostics checks. Health checks are expected to use this size for filling a buffer that is used for creation of digest. | + +## CacheMetricsSpec + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `cache_type` | string | Yes | — | Low-cardinality cache type label for metrics, for example `cas` or `ac`. | +| `backend` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | Store to wrap with cache operation metrics. | + +## MemorySpec + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `eviction_policy` | [EvictionPolicy](#evictionpolicy) | — | — | Policy used to evict items out of the store. Failure to set this value will cause items to never be removed from the store causing infinite memory usage. | + +## ExperimentalCloudObjectSpec + +See [`experimental_cloud_object_store`](#experimental_cloud_object_store-1) for details + +## OntapS3ExistenceCacheSpec + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `index_path` | string | Yes | — | | +| `sync_interval_seconds` | integer (uint32) | Yes | — | | +| `backend` | [ExperimentalOntapS3Spec](#experimentalontaps3spec) | Yes | — | | + +## VerifySpec + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `backend` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | The underlying store wrap around. All content will first flow through self before forwarding to backend. In the event there is an error detected in self, the connection to the backend will be terminated, and early termination should always cause updates to fail on the backend. | +| `verify_size` | boolean | — | `false` | If set the store will verify the size of the data before accepting an upload of data. | +| `verify_hash` | boolean | — | `false` | If the data should be hashed and verify that the key matches the computed hash. The hash function is automatically determined based request and if not set will use the global default. | + +## CompletenessCheckingSpec + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `backend` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | The underlying store that will have it's results validated before sending to client. | +| `cas_store` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | When a request is made, the results are decoded and all output digests/files are verified to exist in this CAS store before returning success. | + +## CompressionSpec + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `backend` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | The underlying store wrap around. All content will first flow through self before forwarding to backend. In the event there is an error detected in self, the connection to the backend will be terminated, and early termination should always cause updates to fail on the backend. | +| `compression_algorithm` | [CompressionAlgorithm](#compressionalgorithm) | Yes | — | The compression algorithm to use. | + +## DedupSpec + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `index_store` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | Store used to store the index of each dedup slice. This store should generally be fast and small. | +| `content_store` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | The store where the individual chunks will be uploaded. This store should generally be the slower & larger store. | +| `min_size` | integer (uint32) | — | 64k | Minimum size that a chunk will be when slicing up the content. Note: This setting can be increased to improve performance because it will actually not check this number of bytes when deciding where to partition the data. | +| `normal_size` | integer (uint32) | — | 256k | A best-effort attempt will be made to keep the average size of the chunks to this number. It is not a guarantee, but a slight attempt will be made. | +| `max_size` | integer (uint32) | — | 512k | Maximum size a chunk is allowed to be. | +| `max_concurrent_fetch_per_get` | integer (uint32) | — | 10 | Due to implementation detail, we want to prefer to download the first chunks of the file so we can stream the content out and free up some of our buffers. This configuration will be used to restrict the number of concurrent chunk downloads at a time per `get()` request. | + +## ExistenceCacheSpec + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `backend` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | The underlying store wrap around. All content will first flow through self before forwarding to backend. In the event there is an error detected in self, the connection to the backend will be terminated, and early termination should always cause updates to fail on the backend. | +| `eviction_policy` | [EvictionPolicy](#evictionpolicy) | — | — | Policy used to evict items out of the store. Failure to set this value will cause items to never be removed from the store causing infinite memory usage. | + +## FastSlowSpec + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `fast` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | Fast store that will be attempted to be contacted before reaching out to the `slow` store. | +| `fast_direction` | [StoreDirection](#storedirection) | — | `"both"` | How to handle the fast store. This can be useful to set to Get for worker nodes such that results are persisted to the slow store only. | +| `slow` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | If the object does not exist in the `fast` store it will try to get it from this store. | +| `slow_direction` | [StoreDirection](#storedirection) | — | `"both"` | How to handle the slow store. This can be useful if creating a diode and you wish to have an upstream read only store. | +| `bypass_dedup_threshold_bytes` | integer (uint64) | — | disabled (0) | Reads of blobs at or above this size skip the leader/follower dedup map and stream straight from the slow store without populating the fast tier. `0` (the default) disables the bypass: every read goes through dedup, matching the prior behaviour. Enable it by setting a threshold — 256 MiB is a reasonable starting point for backends where large-blob dedup is a net loss (followers tend to time out anyway), but the right value is workload-dependent. | + +## ShardSpec + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `stores` | array of [ShardConfig](#shardconfig) | Yes | — | Stores to shard the data to. | + +## FilesystemSpec + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `content_path` | string | Yes | — | Path on the system where to store the actual content. This is where the bulk of the data will be placed. On service startup this folder will be scanned and all files will be added to the cache. In the event one of the files doesn't match the criteria, the file will be deleted. | +| `temp_path` | string | Yes | — | A temporary location of where files that are being uploaded or deleted will be placed while the content cannot be guaranteed to be accurate. This location must be on the same block device as `content_path` so atomic moves can happen (i.e., move without copy). All files in this folder will be deleted on every startup. | +| `read_buffer_size` | integer (uint32) | — | 32k | Buffer size to use when reading files. Generally this should be left to the default value except for testing. | +| `eviction_policy` | [EvictionPolicy](#evictionpolicy) | — | — | Policy used to evict items out of the store. Failure to set this value will cause items to never be removed from the store causing infinite memory usage. | +| `block_size` | integer (uint64) | — | 4kb | The block size of the filesystem for the running machine value is used to determine an entry's actual size on disk consumed For a 4KB block size filesystem, a 1B file actually consumes 4KB | +| `max_concurrent_writes` | integer (uint) | — | unlimited | Maximum number of concurrent write operations allowed. Each write involves streaming data to a temp file and calling `sync_all()`, which can saturate disk I/O when many writes happen simultaneously. Limiting concurrency prevents disk saturation from blocking the async runtime. A value of 0 means unlimited (no concurrency limit). | + +## RefSpec + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `name` | string | Yes | — | Name of the store under the root "stores" config object. | + +## SizePartitioningSpec + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `size` | integer (uint64) | Yes | — | Size to partition the data on. | +| `lower_store` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | Store to send data when object is < (less than) size. | +| `upper_store` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | Store to send data when object is >= (less than eq) size. | + +## GrpcSpec + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `instance_name` | string | — | `""` | Instance name for GRPC calls. Proxy calls will have the `instance_name` changed to this. | +| `endpoints` | array of [GrpcEndpoint](#grpcendpoint) | Yes | — | The endpoint of the grpc connection. | +| `store_type` | [StoreType](#storetype) | Yes | — | The type of the upstream store, this ensures that the correct server calls are made. | +| `retry` | [Retry](#retry) | — | — | Retry configuration to use when a network request fails. | +| `max_concurrent_requests` | integer (uint) | — | `0` | Limit the number of simultaneous upstream requests to this many. A value of zero is treated as unlimited. If the limit is reached the request is queued. | +| `connections_per_endpoint` | integer (uint) | — | `0` | The number of connections to make to each specified endpoint to balance the load over multiple TCP connections. Default 1. | +| `rpc_timeout_s` | integer (uint64) | — | 0 (disabled) | Maximum time (seconds) allowed for a single RPC request (e.g. a `ByteStream.Write` call) before it is cancelled. | +| `use_legacy_resource_names` | boolean | — | false | Use legacy `ByteStream` resource name format, omitting the digest function component from the path. | +| `headers` | map of string to string | — | — | Static headers to attach to every outgoing gRPC request sent to this store's upstream endpoints. Useful for fixed authentication tokens (e.g. `{"authorization": "Bearer "}`) and other static metadata. | +| `forward_headers` | array of string | — | — | Header names to forward from the incoming client request to every outgoing upstream request. The header value is taken from the client request that triggered this store operation. Use this to pass through dynamic credentials such as JWT tokens sent by build clients. | + +## RedisSpec + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `addresses` | array of string | Yes | — | The hostname or IP address of the Redis server. Ex: `["redis://username:password@redis-server-url:6380/99"]` 99 Represents database ID, 6380 represents the port. | +| `response_timeout_s` | integer (uint64) | — | 10 | DEPRECATED: use `command_timeout_ms` The response timeout for the Redis connection in seconds. | +| `connection_timeout_s` | integer (uint64) | — | 10 | DEPRECATED: use `connection_timeout_ms` | +| `experimental_pub_sub_channel` | string | — | (Empty String / No Channel) | An optional and experimental Redis channel to publish write events to. | +| `key_prefix` | string | — | (Empty String / No Prefix) | An optional prefix to prepend to all keys in this store. | +| `mode` | [RedisMode](#redismode) | — | standard, | Set the mode Redis is operating in. | +| `broadcast_channel_capacity` | integer (uint) | — | `0` | Deprecated as redis-rs doesn't use it | +| `command_timeout_ms` | integer (uint64) | — | 10000 (10 seconds) | The amount of time in milliseconds until the Redis store considers the command to be timed out. This will trigger a retry of the command and potentially a reconnection to the Redis server. | +| `connection_timeout_ms` | integer (uint64) | — | 3000 (3 seconds) | The amount of time in milliseconds until the Redis store considers the connection to unresponsive. This will trigger a reconnection to the Redis server. | +| `health_check_timeout_ms` | integer (uint64) | — | 4000 (4 seconds) | Per-call ceiling for the `check_health` PING in milliseconds. | +| `read_chunk_size` | integer (uint) | — | 64KiB | The amount of data to read from the Redis server at a time. This is used to limit the amount of memory used when reading large objects from the Redis server as well as limiting the amount of time a single read operation can take. | +| `connection_pool_size` | integer (uint) | — | 3 | The number of connections to keep open to the Redis servers. | +| `max_chunk_uploads_per_update` | integer (uint) | — | 10 | The maximum number of upload chunks to allow per update. This is used to limit the amount of memory used when uploading large objects to the Redis server. A good rule of thumb is to think of the data as: `AVAIL_MEMORY / (read_chunk_size * max_chunk_uploads_per_update) = THORETICAL_MAX_CONCURRENT_UPLOADS` (note: it is a good idea to divide `AVAIL_MAX_MEMORY` by ~10 to account for other memory usage) | +| `scan_count` | integer (uint) | — | 10000 | The COUNT value passed when scanning keys in Redis. This is used to hint the amount of work that should be done per response. | +| `retry` | [Retry](#retry) | — | — | Retry configuration to use when a network request fails. | +| `max_client_permits` | integer (uint) | — | 500 | Maximum number of permitted actions to the Redis store at any one time This stops problems with timeouts due to many, many inflight actions | +| `max_count_per_cursor` | integer (uint64) | — | 1500 | Maximum number of items returned per cursor for the search indexes May reduce thundering herd issues with worker provisioner at higher node counts, | + +## NoopSpec + +_No fields._ + +## ExperimentalMongoSpec + +Configuration for `ExperimentalMongoDB` store. + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `connection_string` | string | Yes | — | `ExperimentalMongoDB` connection string. Example: <mongodb://localhost:27017> or <mongodb+srv://cluster.mongodb.net> | +| `database` | string | — | "nativelink" | The database name to use. | +| `cas_collection` | string | — | "cas" | The collection name for CAS data. | +| `scheduler_collection` | string | — | "scheduler" | The collection name for scheduler data. | +| `key_prefix` | string | — | "" | Prefix to prepend to all keys stored in `MongoDB`. | +| `read_chunk_size` | integer (uint) | — | 65536 (64KB) | The maximum amount of data to read from `MongoDB` in a single chunk (in bytes). | +| `max_concurrent_uploads` | integer (uint) | — | 10 | Deprecated, unused Maximum number of concurrent uploads allowed. | +| `connection_timeout_ms` | integer (uint64) | — | 3000 | Connection timeout in milliseconds. | +| `command_timeout_ms` | integer (uint64) | — | 10000 | Command timeout in milliseconds. | +| `enable_change_streams` | boolean | — | false | Enable `MongoDB` change streams for real-time updates. Required for scheduler subscriptions. | +| `write_concern_w` | string | — | — | Write concern 'w' parameter. Can be a number (e.g., 1) or string (e.g., "majority"). | +| `write_concern_j` | boolean | — | — | Write concern 'j' parameter (journal acknowledgment). | +| `write_concern_timeout_ms` | integer (uint32) | — | — | Write concern timeout in milliseconds. | +| `max_requests` | integer (uint) | — | Unlimited | Limits the number of requests at any one time | + +## LocalWorkerConfig + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `name` | string | — | {Index position in the workers list} | Name of the worker. This is give a more friendly name to a worker for logging and metric publishing. This is also the prefix of the worker id (i.e., "{name}{uuidv6}"). | +| `worker_api_endpoint` | [EndpointConfig](#endpointconfig) | Yes | — | Endpoint which the worker will connect to the scheduler's `WorkerApiService`. | +| `max_action_timeout_s` | integer (uint) | — | 20 minutes | The maximum time an action is allowed to run. If a task requests for a timeout longer than this time limit, the task will be rejected. Value in seconds. | +| `max_upload_timeout_s` | integer (uint) | — | 10 minutes | Maximum time allowed for uploading action results to CAS after execution completes. If upload takes longer than this, the action fails with `DeadlineExceeded` and may be retried by the scheduler. Value in seconds. | +| `max_cleanup_wait_s` | integer (uint) | — | 30 seconds | Maximum time to wait for action directory cleanup before timing out. Value in seconds. | +| `max_cleanup_backoff_ms` | integer (uint) | — | 500 milliseconds | Maximum backoff duration for exponential backoff when waiting for cleanup. Value in milliseconds. | +| `max_inflight_tasks` | integer (uint64) | — | 0 (infinite tasks) | Maximum number of inflight tasks this worker can cope with. | +| `timeout_handled_externally` | boolean | — | false (`NativeLink` fully handles timeouts) | If timeout is handled in `entrypoint` or another wrapper script. If set to true `NativeLink` will not honor the timeout the action requested and instead will always force kill the action after `max_action_timeout` has been reached. If this is set to false, the smaller value of the action's timeout and `max_action_timeout` will be used to which `NativeLink` will kill the action. | +| `entrypoint` | string | — | {Use the command from the job request} | The command to execute on every execution request. This will be parsed as a command + arguments (not shell). Example: "run.sh" and a job with command: "sleep 5" will result in a command like: "run.sh sleep 5". | +| `experimental_precondition_script` | string | — | — | An optional script to run before every action is processed on the worker. The value should be the full path to the script to execute and will pause all actions on the worker if it returns an exit code other than 0. If not set, then the worker will never pause and will continue to accept jobs according to the scheduler configuration. This is useful, for example, if the worker should not take any more actions until there is enough resource available on the machine to handle them. | +| `cas_fast_slow_store` | string | Yes | — | Underlying CAS store that the worker will use to download CAS artifacts. This store must be a `FastSlowStore`. The `fast` store must be a `FileSystemStore` because it will use hardlinks when building out the files instead of copying the files. The slow store must eventually resolve to the same store the scheduler/client uses to send job requests. | +| `upload_action_result` | [UploadActionResultConfig](#uploadactionresultconfig) | — | — | Configuration for uploading action results. | +| `work_directory` | string | Yes | — | The directory work jobs will be executed from. This directory will be fully managed by the worker service and will be purged on startup. This directory and the directory referenced in `local_filesystem_store_ref`'s `stores::FilesystemStore::content_path` must be on the same filesystem. Hardlinks will be used when placing files that are accessible to the jobs that are sourced from `local_filesystem_store_ref`'s `content_path`. | +| `platform_properties` | map of string to [WorkerProperty](#workerproperty) | Yes | — | Properties of this worker. This configuration will be sent to the scheduler and used to tell the scheduler to restrict what should be executed on this worker. | +| `additional_environment` | map of string to [EnvironmentSource](#environmentsource) | — | — | An optional mapping of environment names to set for the execution as well as those specified in the action itself. If set, will set each key as an environment variable before executing the job with the value of the environment variable being the value of the property of the action being executed of that name or the fixed value. | +| `directory_cache` | [DirectoryCacheConfig](#directorycacheconfig) | — | — | Optional directory cache configuration for improving performance by caching reconstructed input directories and using hardlinks instead of rebuilding them from CAS for every action. | +| `use_namespaces` | boolean | — | False | Whether to use namespaces to isolate the execution. This is only available on Linux. It is highly recommended as it avoids a number of issues with zombie processes and also provides additional hermeticity. If explicitly set to true and it is not supported the worker will exit with an error. | +| `use_mount_namespace` | boolean | — | False | Whether to use a mount namespace to isolate the worker root. This is only available on Linux and when `use_namespaces` is true. It is highly recommended provides additional hermeticity. If explicitly set to true and it is not supported or `use_namespaces` is not set to true the worker will exit with an error. | + +## SimpleSpec + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `supported_platform_properties` | map of string to [PropertyType](#propertytype) | — | — | A list of supported platform properties mapped to how these properties are used when the scheduler looks for worker nodes capable of running the task. | +| `retain_completed_for_s` | integer (uint32) | — | 60 seconds | The amount of time to retain completed actions for in case a `WaitExecution` is called after the action has completed. | +| `client_action_timeout_s` | integer (uint64) | — | 60 seconds | Mark operations as completed with error if no client has updated them within this duration. | +| `worker_timeout_s` | integer (uint64) | — | 5 seconds | Remove workers from pool once the worker has not responded in this amount of time in seconds. | +| `max_action_executing_timeout_s` | integer (uint64) | — | 0 (disabled) | Maximum time (seconds) an action can stay in Executing state without any worker update before being timed out and re-queued. This applies regardless of worker keepalive status, catching cases where a worker is alive (sending keepalives) but stuck on a specific action. Set to 0 to disable (relies only on `worker_timeout_s`). | +| `max_job_retries` | integer (uint) | — | 3 | If a job returns an internal error or times out this many times when attempting to run on a worker the scheduler will return the last error to the client. Jobs will be retried and this configuration is to help prevent one rogue job from infinitely retrying and taking up a lot of resources when the task itself is the one causing the server to go into a bad state. | +| `allocation_strategy` | [WorkerAllocationStrategy](#workerallocationstrategy) | — | `"least_recently_used"` | The strategy used to assign workers jobs. | +| `experimental_backend` | [ExperimentalSimpleSchedulerBackend](#experimentalsimpleschedulerbackend) | — | memory | The storage backend to use for the scheduler. | +| `worker_match_logging_interval_s` | integer (int64) | — | `10` | Every N seconds, do logging of worker matching e.g. "worker busy", "can't find any worker" Defaults to 10s. Can be set to `-1` to disable | + +## SchedulerGrpcSpec + +A scheduler that forwards requests to an upstream scheduler. This +is useful to use when doing some kind of local action cache or CAS away from +the main cluster of workers. In general, it's more efficient to point the +build at the main scheduler directly though. + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `endpoint` | [GrpcEndpoint](#grpcendpoint) | Yes | — | The upstream scheduler to forward requests to. | +| `retry` | [Retry](#retry) | — | — | Retry configuration to use when a network request fails. | +| `max_concurrent_requests` | integer (uint) | — | unlimited | Limit the number of simultaneous upstream requests to this many. A value of zero is treated as unlimited. If the limit is reached the request is queued. | +| `connections_per_endpoint` | integer (uint) | — | 1 | The number of connections to make to each specified endpoint to balance the load over multiple TCP connections. | + +## CacheLookupSpec + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `ac_store` | string | Yes | — | The reference to the action cache store used to return cached actions from rather than running them again. To prevent unintended issues, this store should probably be a `CompletenessCheckingSpec`. | +| `scheduler` | [SchedulerSpec](#schedulerspec) | Yes | — | The nested scheduler to use if cache lookup fails. | + +## PropertyModifierSpec + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `modifications` | array of [PropertyModification](#propertymodification) | Yes | — | A list of modifications to perform to incoming actions for the nested scheduler. These are performed in order and blindly, so removing a property that doesn't exist is fine and overwriting an existing property is also fine. If adding properties that do not exist in the nested scheduler is not supported and will likely cause unexpected behaviour. | +| `scheduler` | [SchedulerSpec](#schedulerspec) | Yes | — | The nested scheduler to use after modifying the properties. | + +## HistoricalResourceSpec + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `hints_file` | string | Yes | — | JSON file containing historical resource hints keyed by Bazel `RequestMetadata` `target_id` and/or `action_mnemonic`. | +| `refresh_interval_s` | integer (uint64) | — | 30 seconds | Reload interval for `hints_file`. Set to 0 to load once. | +| `cpu_property_name` | string | — | `cpu_count` | Platform property name used for CPU minimum values. | +| `memory_property_name` | string | — | `memory_kb` | Platform property name used for memory minimum values, expressed in KiB. | +| `scheduler` | [SchedulerSpec](#schedulerspec) | Yes | — | The nested scheduler to use after applying resource hints. | + +## ListenerConfig + +Plus exactly one of the following variants (the key selects the variant): + +### `http` + +Listener for HTTP/HTTPS/HTTP2 sockets. + +**Type:** [HttpListener](#httplistener) + +## ServicesConfig + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `cas` | array of [CasServiceConfig](#casserviceconfig) | — | — | The Content Addressable Storage (CAS) backend config. The key is the `instance_name` used in the protocol and the value is the underlying CAS store config. | +| `ac` | array of [ActionCacheServiceConfig](#actioncacheserviceconfig) | — | — | The Action Cache (AC) backend config. The key is the `instance_name` used in the protocol and the value is the underlying AC store config. | +| `capabilities` | array of [CapabilitiesServiceConfig](#capabilitiesserviceconfig) | — | — | Capabilities service is required in order to use most of the bazel protocol. This service is used to provide the supported features and versions of this bazel GRPC service. | +| `execution` | array of [ExecutionServiceConfig](#executionserviceconfig) | — | — | The remote execution service configuration. NOTE: This service is under development and is currently just a place holder. | +| `bytestream` | array of [ByteStreamServiceConfig](#bytestreamserviceconfig) | — | — | This is the service used to stream data to and from the CAS. Bazel's protocol strongly encourages users to use this streaming interface to interact with the CAS when the data is large. | +| `fetch` | array of [FetchServiceConfig](#fetchserviceconfig) | — | — | These two are collectively the Remote Asset protocol, but it's defined as two separate services | +| `push` | array of [PushServiceConfig](#pushserviceconfig) | — | — | | +| `worker_api` | [WorkerApiConfig](#workerapiconfig) | — | — | This is the service used for workers to connect and communicate through. NOTE: This service should be served on a different, non-public port. In other words, `worker_api` configuration should not have any other services that are served on the same port. Doing so is a security risk, as workers have a different permission set than a client that makes the remote execution/cache requests. | +| `experimental_bep` | [BepConfig](#bepconfig) | — | — | Experimental - Build Event Protocol (BEP) configuration. This is the service that will consume build events from the client and publish them to a store for processing by an external service. | +| `admin` | [AdminConfig](#adminconfig) | — | — | This is the service for any administrative tasks. It provides a REST API endpoint for administrative purposes. | +| `health` | [HealthConfig](#healthconfig) | — | — | This is the service for health status check. | + +## IdentityHeaderSpec + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `header_name` | string | — | "x-identity" | The name of the header to look for the identity in. | +| `required` | boolean | — | `false` | If the header is required to be set or fail the request. | + +## OriginEventsPublisherSpec + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `store` | string | Yes | — | The store to publish nativelink events to. The store name referenced in the `stores` map in the main config. | + +## ConfigDigestHashFunction + +| Value | Description | +| --- | --- | +| `"sha256"` | Use the sha256 hash function. [https://en.wikipedia.org/wiki/SHA-2](https://en.wikipedia.org/wiki/SHA-2) | +| `"blake3"` | Use the blake3 hash function. [https://en.wikipedia.org/wiki/BLAKE_(hash_function)](https://en.wikipedia.org/wiki/BLAKE_(hash_function)) | + +## EvictionPolicy + +Eviction policy always works on LRU (Least Recently Used). Any time an entry +is touched it updates the timestamp. Inserts and updates will execute the +eviction policy removing any expired entries and/or the oldest entries +until the store size becomes smaller than `max_bytes`. + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `max_bytes` | integer (uint) | — | 0 | Maximum number of bytes before eviction takes place. Zero means never evict based on size. | +| `evict_bytes` | integer (uint) | — | 0 | When eviction starts based on hitting `max_bytes`, continue until `max_bytes - evict_bytes` is met to create a low watermark. This stops operations from thrashing when the store is close to the limit. | +| `max_seconds` | integer (uint32) | — | 0 | Maximum number of seconds for an entry to live since it was last accessed before it is evicted. Zero means never evict based on time. | +| `max_count` | integer (uint64) | — | 0 | Maximum size of the store before an eviction takes place. Zero means never evict based on count. | + +## Retry + +Retry configuration. This configuration is exponential and each iteration +a jitter as a percentage is applied of the calculated delay. For example: +```haskell +Retry{ + max_retries: 7, + delay: 0.1, + jitter: 0.5, +} +``` +will result in: + +| Attempt | Delay | +| --- | --- | +| 1 | 0 ms | +| 2 | 75 to 125 ms | +| 3 | 150 to 250 ms | +| 4 | 300 to 500 ms | +| 5 | 600 ms to 1 s | +| 6 | 1.2 to 2 s | +| 7 | 2.4 to 4 s | +| 8 | 4.8 to 8 s | + +The total delay is additive, so this example produces 9.525 to 15.875 s of total delay for a single request. + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `max_retries` | integer (uint) | — | `0` | Maximum number of retries until retrying stops. Setting this to zero will always attempt 1 time, but not retry. | +| `delay` | number (float) | — | `0` | Delay in seconds for exponential back off. | +| `jitter` | number (float) | — | `0` | Amount of jitter to add as a percentage in decimal form. This will change the formula like: | +| `retry_on_errors` | array of [ErrorCode](#errorcode) | — | — | A list of error codes to retry on, if this is not set then the default error codes to retry on are used. These default codes are the most likely to be non-permanent: `Unknown`; `Cancelled`; `DeadlineExceeded`; `ResourceExhausted`; `Aborted`; `Internal`; `Unavailable`; `DataLoss` | + +## ExperimentalOntapS3Spec + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `endpoint` | string | Yes | — | | +| `vserver_name` | string | Yes | — | | +| `bucket` | string | Yes | — | | +| `root_certificates` | string | — | — | | +| `key_prefix` | string | — | — | If you wish to prefix the location in the bucket. If None, no prefix will be used. | +| `retry` | [Retry](#retry) | — | — | Retry configuration to use when a network request fails. | +| `consider_expired_after_s` | integer (uint32) | — | 0 | If the number of seconds since the `last_modified` time of the object is greater than this value, the object will not be considered "existing". This allows for external tools to delete objects that have not been uploaded in a long time. If a client receives a `NotFound` the client should re-upload the object. | +| `max_retry_buffer_per_request` | integer (uint) | — | 5MB | The maximum buffer size to retain in case of a retryable error during upload. Setting this to zero will disable upload buffering; this means that in the event of a failure during upload, the entire upload will be aborted and the client will likely receive an error. | +| `multipart_max_concurrent_uploads` | integer (uint) | — | 10 | Maximum number of concurrent `UploadPart` requests per `MultipartUpload`. | +| `insecure_allow_http` | boolean | — | false | Allow unencrypted HTTP connections. Only use this for local testing. | +| `disable_http2` | boolean | — | false | Disable HTTP/2 connections and only use HTTP/1.1. Default client configuration will have HTTP/1.1 and HTTP/2 enabled for connection schemes. HTTP/2 should be disabled if environments have poor support or performance related to HTTP/2. Safe to keep default unless underlying network environment, S3, or GCS API servers specify otherwise. | + +## CompressionAlgorithm + +Plus exactly one of the following variants (the key selects the variant): + +### `lz4` + +LZ4 compression algorithm is extremely fast for compression and +decompression, however does not perform very well in compression +ratio. In most cases build artifacts are highly compressible, however +lz4 is quite good at aborting early if the data is not deemed very +compressible. + +see: [https://lz4.github.io/lz4/](https://lz4.github.io/lz4/) + +**Type:** [Lz4Config](#lz4config) + +## StoreDirection + +| Value | Description | +| --- | --- | +| `"both"` | The store operates normally and all get and put operations are handled by it. | +| `"update"` | Update operations will cause persistence to this store, but Get operations will be ignored. This only makes sense on the fast store as the slow store will never get written to on Get anyway. | +| `"get"` | Get operations will cause persistence to this store, but Update operations will be ignored. | +| `"read_only"` | Operate as a read only store, only really makes sense if there's another way to write to it. | + +## ShardConfig + +Configuration for an individual shard of the store. + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `store` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | Store to shard the data to. | +| `weight` | integer (uint32) | — | 1 | The weight of the store. This is used to determine how much data should be sent to the store. The actual percentage is the sum of all the store's weights divided by the individual store's weight. | + +## GrpcEndpoint + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `address` | string | Yes | — | The endpoint address (i.e. grpc://example.com:443 or grpcs://example.com:443). | +| `tls_config` | [ClientTlsConfig](#clienttlsconfig) | — | — | The TLS configuration to use to connect to the endpoint (if grpcs). | +| `concurrency_limit` | integer (uint) | — | — | The maximum concurrency to allow on this endpoint. | +| `connect_timeout_s` | integer (uint64) | — | 30 seconds | Timeout for establishing a TCP connection to the endpoint (seconds). | +| `tcp_keepalive_s` | integer (uint64) | — | 30 seconds | TCP keepalive interval (seconds). Sends TCP keepalive probes at this interval to detect dead connections at the OS level. | +| `http2_keepalive_interval_s` | integer (uint64) | — | 30 seconds | HTTP/2 keepalive interval (seconds). Sends HTTP/2 PING frames at this interval to detect dead connections at the application level. | +| `http2_keepalive_timeout_s` | integer (uint64) | — | 20 seconds | HTTP/2 keepalive timeout (seconds). If a PING response is not received within this duration, the connection is considered dead. | + +## StoreType + +| Value | Description | +| --- | --- | +| `"cas"` | The store is content addressable storage. | +| `"ac"` | The store is an action cache. | + +## RedisMode + +| Value | Description | +| --- | --- | +| `"cluster"` | Use Redis Cluster. | +| `"sentinel"` | Use Redis Sentinel. | +| `"standard"` | Use a standalone Redis server. | + +## EndpointConfig + +Generic config for an endpoint and associated configs. + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `uri` | string | Yes | — | URI of the endpoint. | +| `timeout` | number (float) | — | 5 seconds | Timeout in seconds that a request should take. | +| `tls_config` | [ClientTlsConfig](#clienttlsconfig) | — | — | The TLS configuration to use to connect to the endpoint. | + +## UploadActionResultConfig + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `ac_store` | string | — | {No uploading is done} | Underlying AC store that the worker will use to publish execution results into. Objects placed in this store should be reachable from the scheduler/client-cas after they have finished updating. | +| `upload_ac_results_strategy` | [UploadCacheResultsStrategy](#uploadcacheresultsstrategy) | — | `SuccessOnly` | In which situations should the results be published to the `ac_store`, if set to `SuccessOnly` then only results with an exit code of 0 will be uploaded, if set to Everything all completed results will be uploaded. | +| `historical_results_store` | string | — | {CAS store of parent} | Store to upload historical results to. This should be a CAS store if set. | +| `upload_historical_results_strategy` | [UploadCacheResultsStrategy](#uploadcacheresultsstrategy) | — | `FailuresOnly` | In which situations should the results be published to the historical CAS. The historical CAS is where failures are published. These messages conform to the CAS key-value lookup format and are always a `HistoricalExecuteResponse` serialized message. | +| `success_message_template` | string | — | "" (no message) | Template to use for the `ExecuteResponse.message` property. This message is attached to the response before it is sent to the client. The following special variables are supported:; `digest_function`: Digest function used to calculate the action digest: `action_digest_hash`: Action digest hash: `action_digest_size`: Action digest size: `historical_results_hash`: `HistoricalExecuteResponse` digest hash: `historical_results_size`: `HistoricalExecuteResponse` digest size. | +| `failure_message_template` | string | — | "" (no message) | Same as `success_message_template` but for failure case. | + +## WorkerProperty + +Plus exactly one of the following variants (the key selects the variant): + +### `values` + +List of static values. +Note: Generally there should only ever be 1 value, but if the platform +property key is `PropertyType::Priority` it may have more than one value. + +**Type:** array of string + +### `query_cmd` + +A dynamic configuration. The string will be executed as a command +(not shell) and will be split by "\n" (new line character). + +**Type:** string + +## EnvironmentSource + +One of: + +- `property`: string — The name of the platform property in the action to get the value from. +- `value`: string — The raw value to set. +- `"from_environment"` — Take the value from the local environment corresponding to the name key +- `"timeout_millis"` — The max amount of time in milliseconds the command is allowed to run (requested by the client). +- `"side_channel_file"` — A special file path will be provided that can be used to communicate with the parent process about out-of-band information. This file will be read after the command has finished executing. Based on the contents of the file, the behavior of the result may be modified. +- `"action_directory"` — A "root" directory for the action. This directory can be used to store temporary files that are not needed after the action has completed. This directory will be purged after the action has completed. + +## DirectoryCacheConfig + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `max_entries` | integer (uint) | — | 1000 | Maximum number of cached directories. | +| `max_size_bytes` | integer (uint64) | — | 10737418240 (10 GB) | Maximum total size in bytes for all cached directories (0 = unlimited). | +| `cache_root` | string | — | `{work_directory}/../directory_cache` | Base directory for cache storage. This directory will be managed by the worker and should be on the same filesystem as `work_directory`. | + +## PropertyType + +When the scheduler matches tasks to workers that are capable of running +the task, this value will be used to determine how the property is treated. + +One of: + +- string +- `"minimum"` — Requires the platform property to be a u64 and when the scheduler looks for appropriate worker nodes that are capable of executing the task, the task will not run on a node that has less than this value. +- `"exact"` — Requires the platform property to be a string and when the scheduler looks for appropriate worker nodes that are capable of executing the task, the task will not run on a node that does not have this property set to the value with exact string match. +- `"priority"` — Does not restrict on this value and instead will be passed to the worker as an informational piece. In the future this will be used by the scheduler and worker to cause the scheduler to prefer certain workers over others, but not restrict them based on these values. + +## WorkerAllocationStrategy + +When a worker is being searched for to run a job, this will be used +on how to choose which worker should run the job when multiple +workers are able to run the task. + +| Value | Description | +| --- | --- | +| `"least_recently_used"` | Prefer workers that have been least recently used to run a job. | +| `"most_recently_used"` | Prefer workers that have been most recently used to run a job. | + +## ExperimentalSimpleSchedulerBackend + +One of: + +- `"memory"` — Use an in-memory store for the scheduler. +- `redis`: [ExperimentalRedisSchedulerBackend](#experimentalredisschedulerbackend) — Use a Redis store for the scheduler. + +## SchedulerSpec + +Plus exactly one of the following variants (the key selects the variant): + +### `simple` + +**Type:** [SimpleSpec](#simplespec) + +### `grpc` + +**Type:** [SchedulerGrpcSpec](#schedulergrpcspec) + +### `cache_lookup` + +**Type:** [CacheLookupSpec](#cachelookupspec) + +### `property_modifier` + +**Type:** [PropertyModifierSpec](#propertymodifierspec) + +### `historical_resource` + +**Type:** [HistoricalResourceSpec](#historicalresourcespec) + +## PropertyModification + +Plus exactly one of the following variants (the key selects the variant): + +### `add` + +Add a property to the action properties. + +**Type:** [PlatformPropertyAddition](#platformpropertyaddition) + +### `remove` + +Remove a named property from the action. + +**Type:** string + +### `replace` + +If a property is found, then replace it with another one. + +**Type:** [PlatformPropertyReplacement](#platformpropertyreplacement) + +## HttpListener + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `socket_address` | string | Yes | — | Address to listen on. Example: `127.0.0.1:8080` or `:8080` to listen to all IPs. | +| `freebind` | boolean | — | false | Allow binding `socket_address` before it is assigned locally. | +| `compression` | [HttpCompressionConfig](#httpcompressionconfig) | — | — | Data transport compression configuration to use for this service. | +| `advanced_http` | [HttpServerConfig](#httpserverconfig) | — | — | Advanced HTTP server configuration. | +| `max_decoding_message_size` | integer (uint) | — | 4 MiB | Maximum number of bytes to decode on each grpc stream chunk. | +| `tls` | [TlsConfig](#tlsconfig) | — | — | TLS configuration for this server. If not set, the server will not use TLS. | + +## CasServiceConfig + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `instance_name` | string | — | `""` | Used when the config references `instance_name` in the protocol. | +| `cas_store` | string | Yes | — | The store name referenced in the `stores` map in the main config. This store name referenced here may be reused multiple times. | + +## ActionCacheServiceConfig + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `instance_name` | string | — | `""` | Used when the config references `instance_name` in the protocol. | +| `ac_store` | string | Yes | — | The store name referenced in the `stores` map in the main config. This store name referenced here may be reused multiple times. | +| `read_only` | boolean | — | `false` | Whether the Action Cache store may be written to, this if set to false it is only possible to read from the Action Cache. | + +## CapabilitiesServiceConfig + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `instance_name` | string | — | `""` | Used when the config references `instance_name` in the protocol. | +| `remote_execution` | [CapabilitiesRemoteExecutionConfig](#capabilitiesremoteexecutionconfig) | — | — | Configuration for remote execution capabilities. If not set the capabilities service will inform the client that remote execution is not supported. | + +## ExecutionServiceConfig + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `instance_name` | string | — | `""` | Used when the config references `instance_name` in the protocol. | +| `cas_store` | string | Yes | — | The store name referenced in the `stores` map in the main config. This store name referenced here may be reused multiple times. This value must be a CAS store reference. | +| `scheduler` | string | Yes | — | The scheduler name referenced in the `schedulers` map in the main config. | + +## ByteStreamServiceConfig + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `instance_name` | string | — | `""` | Used when the config references `instance_name` in the protocol. | +| `cas_store` | string | Yes | — | Name of the store in the "stores" configuration. | +| `max_bytes_per_stream` | integer (uint) | — | 64KiB | Max number of bytes to send on each grpc stream chunk. According to [https://github.com/grpc/grpc.github.io/issues/371](https://github.com/grpc/grpc.github.io/issues/371) 16KiB - 64KiB is optimal. | +| `persist_stream_on_disconnect_timeout_s` | integer (uint) | — | 10 seconds | In the event a client disconnects while uploading a blob, we will hold the internal stream open for this many seconds before closing it. This allows clients that disconnect to reconnect and continue uploading the same blob. | + +## FetchServiceConfig + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `instance_name` | string | — | `""` | Used when the config references `instance_name` in the protocol. | +| `fetch_store` | string | Yes | — | The store name referenced in the `stores` map in the main config. This store name referenced here may be reused multiple times. | + +## PushServiceConfig + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `instance_name` | string | — | `""` | Used when the config references `instance_name` in the protocol. | +| `push_store` | string | Yes | — | The store name referenced in the `stores` map in the main config. This store name referenced here may be reused multiple times. | +| `read_only` | boolean | — | `false` | Whether the Action Cache store may be written to, this if set to false it is only possible to read from the Action Cache. | + +## WorkerApiConfig + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `scheduler` | string | Yes | — | The scheduler name referenced in the `schedulers` map in the main config. | + +## BepConfig + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `store` | string | Yes | — | The store to publish build events to. The store name referenced in the `stores` map in the main config. | + +## AdminConfig + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `path` | string | — | "/admin" | Path to register the admin API. If path is "/admin", and your domain is "example.com", you can reach the endpoint with: [http://example.com/admin](http://example.com/admin). | + +## HealthConfig + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `path` | string | — | "/status" | Path to register the health status check. If path is "/status", and your domain is "example.com", you can reach the endpoint with: [http://example.com/status](http://example.com/status). | +| `timeout_seconds` | integer (uint64) | — | 5s | Timeout on health checks. | + +## ErrorCode + +The possible error codes that might occur on an upstream request. + +| Value | Description | +| --- | --- | +| `"Cancelled"` | | +| `"Unknown"` | | +| `"InvalidArgument"` | | +| `"DeadlineExceeded"` | | +| `"NotFound"` | | +| `"AlreadyExists"` | | +| `"PermissionDenied"` | | +| `"ResourceExhausted"` | | +| `"FailedPrecondition"` | | +| `"Aborted"` | | +| `"OutOfRange"` | | +| `"Unimplemented"` | | +| `"Internal"` | | +| `"Unavailable"` | | +| `"DataLoss"` | | +| `"Unauthenticated"` | | + +## Lz4Config + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `block_size` | integer (uint32) | — | 65536 (64k) | Size of the blocks to compress. Higher values require more ram, but might yield slightly better compression ratios. | +| `max_decode_block_size` | integer (uint32) | — | value in `block_size` | Maximum size allowed to attempt to deserialize data into. This is needed because the `block_size` is embedded into the data so if there was a bad actor, they could upload an extremely large `block_size`'ed entry and we'd allocate a large amount of memory when retrieving the data. To prevent this from happening, we allow you to specify the maximum that we'll attempt to deserialize. | + +## ClientTlsConfig + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `ca_file` | string | — | — | Path to the certificate authority to use to validate the remote. | +| `cert_file` | string | — | — | Path to the certificate file for client authentication. | +| `key_file` | string | — | — | Path to the private key file for client authentication. | +| `use_native_roots` | boolean | — | false | If set the client will use the native roots for TLS connections. | + +## UploadCacheResultsStrategy + +| Value | Description | +| --- | --- | +| `"success_only"` | Only upload action results with an exit code of 0. | +| `"never"` | Don't upload any action results. | +| `"everything"` | Upload all action results that complete. | +| `"failures_only"` | Only upload action results that fail. | + +## ExperimentalRedisSchedulerBackend + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `redis_store` | string | Yes | — | A reference to the Redis store to use for the scheduler. Note: This MUST resolve to a `RedisSpec`. | + +## PlatformPropertyAddition + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `name` | string | Yes | — | The name of the property to add. | +| `value` | string | Yes | — | The value to assign to the property. | + +## PlatformPropertyReplacement + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `name` | string | Yes | — | The name of the property to replace. | +| `value` | string | — | — | The value to match against, if unset then any instance matches. | +| `new_name` | string | Yes | — | The new name of the property. | +| `new_value` | string | — | — | The value to assign to the property, if unset will remain the same. | + +## HttpCompressionConfig + +Note: Compressing data in the cloud rarely has a benefit, since most +cloud providers have very high bandwidth backplanes. However, for +clients not inside the data center, it might be a good idea to +compress data to and from the cloud. This will however come at a high +CPU and performance cost. If you are making remote execution share the +same CAS/AC servers as client's remote cache, you can create multiple +services with different compression settings that are served on +different ports. Then configure the non-cloud clients to use one port +and cloud-clients to use another. + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `send_compression_algorithm` | [HttpCompressionAlgorithm](#httpcompressionalgorithm) | — | `HttpCompressionAlgorithm::None` | The compression algorithm that the server will use when sending responses to clients. Enabling this will likely save a lot of data transfer, but will consume a lot of CPU and add a lot of latency. see: [https://github.com/tracemachina/nativelink/issues/109](https://github.com/tracemachina/nativelink/issues/109) | +| `accepted_compression_algorithms` | array of [HttpCompressionAlgorithm](#httpcompressionalgorithm) | Yes | {no supported compression} | The compression algorithm that the server will accept from clients. The server will broadcast the supported compression algorithms to clients and the client will choose which compression algorithm to use. Enabling this will likely save a lot of data transfer, but will consume a lot of CPU and add a lot of latency. see: [https://github.com/tracemachina/nativelink/issues/109](https://github.com/tracemachina/nativelink/issues/109) | + +## HttpServerConfig + +Advanced HTTP configuration. These generally should not be set. +For documentation on these settings, see the hyper documentation: +See: [hyper HTTP server docs](https://docs.rs/hyper/latest/hyper/server/conn/struct.Http.html) + +Note: All of these default to the default values from hyper unless otherwise +specified. + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `http2_keep_alive_interval` | integer (uint32) | — | — | Interval to send keep-alive pings via HTTP2. Note: This is in seconds. | +| `experimental_http2_max_pending_accept_reset_streams` | integer (uint32) | — | — | | +| `experimental_http2_initial_stream_window_size` | integer (uint32) | — | — | | +| `experimental_http2_initial_connection_window_size` | integer (uint32) | — | — | | +| `experimental_http2_adaptive_window` | boolean | — | — | | +| `experimental_http2_max_frame_size` | integer (uint32) | — | — | | +| `experimental_http2_max_concurrent_streams` | integer (uint32) | — | — | | +| `experimental_http2_keep_alive_timeout_s` | integer (uint32) | — | — | | +| `experimental_http2_max_send_buf_size` | integer (uint32) | — | — | | +| `experimental_http2_enable_connect_protocol` | boolean | — | — | | +| `experimental_http2_max_header_list_size` | integer (uint32) | — | — | | + +## TlsConfig + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `cert_file` | string | Yes | — | Path to the certificate file. | +| `key_file` | string | Yes | — | Path to the private key file. | +| `client_ca_file` | string | — | — | Path to the certificate authority for mTLS, if client authentication is required for this endpoint. | +| `client_crl_file` | string | — | — | Path to the certificate revocation list for mTLS, if client authentication is required for this endpoint. | + +## CapabilitiesRemoteExecutionConfig + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `scheduler` | string | Yes | — | Scheduler used to configure the capabilities of remote execution. | + +## HttpCompressionAlgorithm + +| Value | Description | +| --- | --- | +| `"none"` | No compression. | +| `"gzip"` | Zlib compression. | + +## Reading the source + +If anything here disagrees with the binary, the source wins: + +- [`stores.rs`](https://github.com/TraceMachina/nativelink/tree/v1.6.0/nativelink-config/src/stores.rs) +- [`cas_server.rs`](https://github.com/TraceMachina/nativelink/tree/v1.6.0/nativelink-config/src/cas_server.rs) +- [`schedulers.rs`](https://github.com/TraceMachina/nativelink/tree/v1.6.0/nativelink-config/src/schedulers.rs) diff --git a/web/apps/docs/content/docs/reference/nativelink-config/v1.6.1.mdx b/web/apps/docs/content/docs/reference/nativelink-config/v1.6.1.mdx new file mode 100644 index 000000000..e6c7307c2 --- /dev/null +++ b/web/apps/docs/content/docs/reference/nativelink-config/v1.6.1.mdx @@ -0,0 +1,1521 @@ +--- +title: Configuration reference +description: Every knob in the NativeLink JSON5 configuration — types, defaults, and links to source, autogenerated from the Rust config crate. +full: true +--- + +{/* AUTOGENERATED — do not edit by hand. + Source: nativelink-config @ v1.6.1 (23e960dc) + Regenerate from web/: bun --filter @nativelink/docs gen:config-reference */} + + + +This is the canonical NativeLink configuration reference for **v1.6.1**. +It is autogenerated from the Rust config crate +([`nativelink-config/src`](https://github.com/TraceMachina/nativelink/tree/v1.6.1/nativelink-config/src)) via the `build-schema` binary, so +it can never drift from what the binary actually deserializes. + +## Top-level fields + +The root object (`CasConfig`) accepts the following fields: + +| Field | Type | Required | Description | +| --- | --- | --- | --- | +| `stores` | array of [NamedStoreConfig](#namedstoreconfig) | Yes | List of stores available to use in this config. The keys can be used in other configs when needing to reference a store. | +| `workers` | array of [WorkerConfig](#workerconfig) | — | Worker configurations used to execute jobs. | +| `schedulers` | array of [NamedSchedulerConfig](#namedschedulerconfig) | — | List of schedulers available to use in this config. The keys can be used in other configs when needing to reference a scheduler. | +| `servers` | array of [ServerConfig](#serverconfig) | Yes | Servers to setup for this process. | +| `experimental_origin_events` | [OriginEventsSpec](#origineventsspec) | — | Experimental - Origin events configuration. This is the service that will collect and publish nativelink events to a store for processing by an external service. | +| `global` | [GlobalConfig](#globalconfig) | — | Any global configurations that apply to all modules live here. | + +## Configuration types + +Every type reachable from the root configuration, in reading order. + +## NamedStoreConfig + +**Common fields** + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `name` | string | Yes | — | | + +Plus exactly one of the following variants (the key selects the variant): + +### `cache_metrics` + +Cache metrics store wraps another store and emits low-cardinality +OpenTelemetry cache operation metrics for the wrapped store. + +This wrapper is opt-in. Stores that are not explicitly wrapped by +`cache_metrics` are constructed exactly as they are without this +wrapper and do not pay its hot-path timing or recording cost. + +**Example JSON5 config:** +```json5 +"cache_metrics": { + "cache_type": "cas", + "backend": { + "filesystem": { + "content_path": "~/.cache/nativelink/content_path-cas", + "temp_path": "~/.cache/nativelink/tmp_path-cas" + } + } +} +``` + +**Type:** [CacheMetricsSpec](#cachemetricsspec) + +### `memory` + +Memory store will store all data in a hash map in memory. + +**Example JSON5 config:** +```json5 +"memory": { + "eviction_policy": { + "max_bytes": "10mb", + } +} +``` + +**Type:** [MemorySpec](#memoryspec) + +### `experimental_cloud_object_store` + +A generic blob store that will store files on the cloud +provider. This configuration will never delete files, so you are +responsible for purging old files in other ways. +It supports the following backends: + +1. **Amazon S3:** + S3 store will use Amazon's S3 service as a backend to store + the files. This configuration can be used to share files + across multiple instances. Uses system certificates for TLS + verification via `rustls-platform-verifier`. + + **Example JSON5 config:** + ```json5 + "experimental_cloud_object_store": { + "provider": "aws", + "region": "eu-north-1", + "bucket": "crossplane-bucket-af79aeca9", + "key_prefix": "test-prefix-index/", + "retry": { + "max_retries": 6, + "delay": 0.3, + "jitter": 0.5 + }, + "multipart_max_concurrent_uploads": 10 + } + ``` + +2. **Google Cloud Storage:** + GCS store uses Google's GCS service as a backend to store + the files. This configuration can be used to share files + across multiple instances. + + **Example JSON5 config:** + ```json5 + "experimental_cloud_object_store": { + "provider": "gcs", + "bucket": "test-bucket", + "key_prefix": "test-prefix-index/", + "retry": { + "max_retries": 6, + "delay": 0.3, + "jitter": 0.5 + }, + "multipart_max_concurrent_uploads": 10 + } + ``` + +3. **Azure Blob Store:** + Azure Blob store will use Microsoft's Azure Blob service as a + backend to store the files. This configuration can be used to + share files across multiple instances. + + **Example JSON5 config:** + ```json5 + "experimental_cloud_object_store": { + "provider": "azure", + "account_name": "cloudshell1393657559", + "container": "simple-test-container", + "key_prefix": "folder/", + "retry": { + "max_retries": 6, + "delay": 0.3, + "jitter": 0.5 + }, + "multipart_max_concurrent_uploads": 10 + } + ``` + +4. **NetApp ONTAP S3:** + NetApp ONTAP S3 store will use ONTAP's S3-compatible storage as a backend + to store files. This store is specifically configured for ONTAP's S3 requirements + including custom TLS configuration, credentials management, and proper vserver + configuration. + + This store uses AWS environment variables for credentials: + - `AWS_ACCESS_KEY_ID` + - `AWS_SECRET_ACCESS_KEY` + - `AWS_DEFAULT_REGION` + + **Example JSON5 config:** + ```json5 + "experimental_cloud_object_store": { + "provider": "ontap", + "endpoint": "https://ontap-s3-endpoint:443", + "vserver_name": "your-vserver", + "bucket": "your-bucket", + "root_certificates": "/path/to/certs.pem", // Optional + "key_prefix": "test-prefix/", // Optional + "retry": { + "max_retries": 6, + "delay": 0.3, + "jitter": 0.5 + }, + "multipart_max_concurrent_uploads": 10 + } + ``` + +5. **Cloudflare R2:** + R2 store uses Cloudflare's R2 service as a backend. R2 speaks the + S3 API, so this is a thin wrapper that derives the account-scoped + endpoint (`https://{account_id}.r2.cloudflarestorage.com`) for you. + + **Example JSON5 config:** + ```json5 + "experimental_cloud_object_store": { + "provider": "r2", + "account_id": "a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4", + "bucket": "nativelink-cas", + "key_prefix": "test-prefix/", + "retry": { + "max_retries": 6, + "delay": 0.3, + "jitter": 0.5 + }, + "multipart_max_concurrent_uploads": 10 + } + ``` + +6. **Oracle Cloud Infrastructure (OCI) Object Storage:** + OCI store uses Oracle Cloud Infrastructure's S3-compatible Object + Storage API. The path-style endpoint is derived from your Object + Storage `namespace` and `region` as + `https://{namespace}.compat.objectstorage.{region}.oci.customer-oci.com`. + Authenticate with a Customer Secret Key (Access Key/Secret Key pair + created under User Settings -> Customer secret keys in the OCI + console); the secret cannot be retrieved after generation, so read + it from an env var via shellexpand. + + **Example JSON5 config:** + ```json5 + "experimental_cloud_object_store": { + "provider": "oci", + "namespace": "your-object-storage-namespace", + "region": "us-phoenix-1", + "bucket": "nativelink-cas", + "access_key_id": "oci_access_key_id", + "secret_access_key": "oci_secret_access_key", + "key_prefix": "test-prefix/", + "retry": { + "max_retries": 6, + "delay": 0.3, + "jitter": 0.5 + } + } + ``` + +**Type:** [ExperimentalCloudObjectSpec](#experimentalcloudobjectspec) + +### `ontap_s3_existence_cache` + +ONTAP S3 Existence Cache provides a caching layer on top of the ONTAP S3 store +to optimize repeated existence checks. It maintains an in-memory cache of object +digests and periodically syncs this cache to disk for persistence. + +The cache helps reduce latency for repeated calls to check object existence, +while still ensuring eventual consistency with the underlying ONTAP S3 store. + +Example JSON5 config: +```json5 +"ontap_s3_existence_cache": { + "index_path": "/path/to/cache/index.json", + "sync_interval_seconds": 300, + "backend": { + "endpoint": "https://ontap-s3-endpoint:443", + "vserver_name": "your-vserver", + "bucket": "your-bucket", + "key_prefix": "test-prefix/" + } +} +``` + +**Type:** [OntapS3ExistenceCacheSpec](#ontaps3existencecachespec) + +### `verify` + +Verify store is used to apply verifications to an underlying +store implementation. It is strongly encouraged to validate +as much data as you can before accepting data from a client, +failing to do so may cause the data in the store to be +populated with invalid data causing all kinds of problems. + +The suggested configuration is to have the CAS validate the +hash and size and the AC validate nothing. + +**Example JSON5 config:** +```json5 +"verify": { + "backend": { + "memory": { + "eviction_policy": { + "max_bytes": "500mb" + } + }, + }, + "verify_size": true, + "verify_hash": true +} +``` + +**Type:** [VerifySpec](#verifyspec) + +### `completeness_checking` + +Completeness checking store verifies if the +output files & folders exist in the CAS before forwarding +the request to the underlying store. +Note: This store should only be used on AC stores. + +**Example JSON5 config:** +```json5 +"completeness_checking": { + "backend": { + "filesystem": { + "content_path": "~/.cache/nativelink/content_path-ac", + "temp_path": "~/.cache/nativelink/tmp_path-ac", + "eviction_policy": { + "max_bytes": "500mb", + } + } + }, + "cas_store": { + "ref_store": { + "name": "CAS_MAIN_STORE" + } + } +} +``` + +**Type:** [CompletenessCheckingSpec](#completenesscheckingspec) + +### `compression` + +A compression store that will compress the data inbound and +outbound. There will be a non-trivial cost to compress and +decompress the data, but in many cases if the final store is +a store that requires network transport and/or storage space +is a concern it is often faster and more efficient to use this +store before those stores. + +**Example JSON5 config:** +```json5 +"compression": { + "compression_algorithm": { + "lz4": {} + }, + "backend": { + "filesystem": { + "content_path": "/tmp/nativelink/data/content_path-cas", + "temp_path": "/tmp/nativelink/data/tmp_path-cas", + "eviction_policy": { + "max_bytes": "2gb", + } + } + } +} +``` + +**Type:** [CompressionSpec](#compressionspec) + +### `dedup` + +A dedup store will take the inputs and run a rolling hash +algorithm on them to slice the input into smaller parts then +run a sha256 algorithm on the slice and if the object doesn't +already exist, upload the slice to the `content_store` using +a new digest of just the slice. Once all parts exist, an +Action-Cache-like digest will be built and uploaded to the +`index_store` which will contain a reference to each +chunk/digest of the uploaded file. Downloading a request will +first grab the index from the `index_store`, and forward the +download content of each chunk as if it were one file. + +This store is exceptionally good when the following conditions +are met: +* Content is mostly the same (inserts, updates, deletes are ok) +* Content is not compressed or encrypted +* Uploading or downloading from `content_store` is the bottleneck. + +Note: This store pairs well when used with `CompressionSpec` as +the `content_store`, but never put `DedupSpec` as the backend of +`CompressionSpec` as it will negate all the gains. + +Note: When running `.has()` on this store, it will only check +to see if the entry exists in the `index_store` and not check +if the individual chunks exist in the `content_store`. + +**Example JSON5 config:** +```json5 +"dedup": { + "index_store": { + "memory": { + "eviction_policy": { + "max_bytes": "1GB", + } + } + }, + "content_store": { + "compression": { + "compression_algorithm": { + "lz4": {} + }, + "backend": { + "fast_slow": { + "fast": { + "memory": { + "eviction_policy": { + "max_bytes": "500MB", + } + } + }, + "slow": { + "filesystem": { + "content_path": "/tmp/nativelink/data/content_path-content", + "temp_path": "/tmp/nativelink/data/tmp_path-content", + "eviction_policy": { + "max_bytes": "2gb" + } + } + } + } + } + } + } +} +``` + +**Type:** [DedupSpec](#dedupspec) + +### `existence_cache` + +Existence store will wrap around another store and cache calls +to has so that subsequent `has_with_results` calls will be +faster. This is useful for cases when you have a store that +is slow to respond to has calls. +Note: This store should only be used on CAS stores. + +**Example JSON5 config:** +```json5 +"existence_cache": { + "backend": { + "memory": { + "eviction_policy": { + "max_bytes": "500mb", + } + } + }, + // Note this is the existence store policy, not the backend policy + "eviction_policy": { + "max_seconds": 100, + } +} +``` + +**Type:** [ExistenceCacheSpec](#existencecachespec) + +### `fast_slow` + +`FastSlow` store will first try to fetch the data from the `fast` +store and then if it does not exist try the `slow` store. +When the object does exist in the `slow` store, it will copy +the data to the `fast` store while returning the data. +This store should be thought of as a store that "buffers" +the data to the `fast` store. +On uploads it will mirror data to both `fast` and `slow` stores. + +WARNING: If you need data to always exist in the `slow` store +for something like remote execution, be careful because this +store will never check to see if the objects exist in the +`slow` store if it exists in the `fast` store (i.e., it assumes +that if an object exists in the `fast` store it will exist in +the `slow` store). + +***Example JSON5 config:*** +```json5 +"fast_slow": { + "fast": { + "filesystem": { + "content_path": "/tmp/nativelink/data/content_path-index", + "temp_path": "/tmp/nativelink/data/tmp_path-index", + "eviction_policy": { + "max_bytes": "500mb", + } + } + }, + "slow": { + "filesystem": { + "content_path": "/tmp/nativelink/data/content_path-index", + "temp_path": "/tmp/nativelink/data/tmp_path-index", + "eviction_policy": { + "max_bytes": "500mb", + } + } + } +} +``` + +**Type:** [FastSlowSpec](#fastslowspec) + +### `shard` + +Shards the data to multiple stores. This is useful for cases +when you want to distribute the load across multiple stores. +The digest hash is used to determine which store to send the +data to. + +**Example JSON5 config:** +```json5 +"shard": { + "stores": [ + { + "store": { + "memory": { + "eviction_policy": { + "max_bytes": "10mb" + }, + }, + }, + "weight": 1 + }] +} +``` + +**Type:** [ShardSpec](#shardspec) + +### `filesystem` + +Stores the data on the filesystem. This store is designed for +local persistent storage. Restarts of this program should restore +the previous state, meaning anything uploaded will be persistent +as long as the filesystem integrity holds. + +**Example JSON5 config:** +```json5 +"filesystem": { + "content_path": "/tmp/nativelink/data-worker-test/content_path-cas", + "temp_path": "/tmp/nativelink/data-worker-test/tmp_path-cas", + "eviction_policy": { + "max_bytes": "10gb", + } +} +``` + +**Type:** [FilesystemSpec](#filesystemspec) + +### `ref_store` + +Store used to reference a store in the root store manager. +This is useful for cases when you want to share a store in different +nested stores. Example, you may want to share the same memory store +used for the action cache, but use a `FastSlowSpec` and have the fast +store also share the memory store for efficiency. + +**Example JSON5 config:** +```json5 +"ref_store": { + "name": "FS_CONTENT_STORE" +} +``` + +**Type:** [RefSpec](#refspec) + +### `size_partitioning` + +Uses the size field of the digest to separate which store to send the +data. This is useful for cases when you'd like to put small objects +in one store and large objects in another store. This should only be +used if the size field is the real size of the content, in other +words, don't use on AC (Action Cache) stores. Any store where you can +safely use `VerifySpec.verify_size = true`, this store should be safe +to use (i.e., CAS stores). + +**Example JSON5 config:** +```json5 +"size_partitioning": { + "size": "128mib", + "lower_store": { + "memory": { + "eviction_policy": { + "max_bytes": "${NATIVELINK_CAS_MEMORY_CONTENT_LIMIT:-100mb}" + } + } + }, + "upper_store": { + /// This store discards data larger than 128mib. + "noop": {} + } +} +``` + +**Type:** [SizePartitioningSpec](#sizepartitioningspec) + +### `grpc` + +This store will pass-through calls to another GRPC store. This store +is not designed to be used as a sub-store of another store, but it +does satisfy the interface and will likely work. + +One major GOTCHA is that some stores use a special function on this +store to get the size of the underlying object, which is only reliable +when this store is serving the a CAS store, not an AC store. If using +this store directly without being a child of any store there are no +side effects and is the most efficient way to use it. + +**Example JSON5 config:** +```json5 +"grpc": { + "instance_name": "main", + "endpoints": [ + {"address": "grpc://${CAS_ENDPOINT:-127.0.0.1}:50051"} + ], + "connections_per_endpoint": "5", + "rpc_timeout_s": "5m", + "store_type": "ac", + // Static headers attached to every outgoing request to the upstream + // remote cache. Useful for fixed service-account credentials. + "headers": { + "authorization": "Bearer my-static-token" + }, + // Header names to copy from the inbound client request and forward to + // the upstream remote cache. Use this to pass through dynamic + // credentials such as a JWT sent by the build client. + "forward_headers": ["authorization", "x-custom-token"] +} +``` + +**Type:** [GrpcSpec](#grpcspec) + +### `redis_store` + +Stores data in any stores compatible with Redis APIs. + +Pairs well with `SizePartitioning` and/or `FastSlow` stores. +Ideal for accepting small object sizes as most Redis store +services have a max file upload of between 256Mb-512Mb. + +**Example JSON5 config:** +```json5 +"redis_store": { + "addresses": [ + "redis://127.0.0.1:6379/", + ], + "max_client_permits": 1000, +} +``` + +**Type:** [RedisSpec](#redisspec) + +### `noop` + +Noop store is a store that sends streams into the void and all data +retrieval will return 404 (`NotFound`). This can be useful for cases +where you may need to partition your data and part of your data needs +to be discarded. + +**Example JSON5 config:** +```json5 +"noop": {} +``` + +**Type:** [NoopSpec](#noopspec) + +### `experimental_mongo` + +Experimental `MongoDB` store implementation. + +This store uses `MongoDB` as a backend for storing data. It supports +both CAS (Content Addressable Storage) and scheduler data with +optional change streams for real-time updates. + +**Example JSON5 config:** +```json5 +"experimental_mongo": { + "connection_string": "mongodb://localhost:27017", + "database": "nativelink", + "cas_collection": "cas", + "key_prefix": "cas:", + "read_chunk_size": 65536, + "max_concurrent_uploads": 10, + "enable_change_streams": false, + "max_requests": "100" +} +``` + +**Type:** [ExperimentalMongoSpec](#experimentalmongospec) + +## WorkerConfig + +Plus exactly one of the following variants (the key selects the variant): + +### `local` + +A worker type that executes jobs locally on this machine. + +**Type:** [LocalWorkerConfig](#localworkerconfig) + +## NamedSchedulerConfig + +**Common fields** + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `name` | string | Yes | — | | + +Plus exactly one of the following variants (the key selects the variant): + +### `simple` + +**Type:** [SimpleSpec](#simplespec) + +### `grpc` + +**Type:** [SchedulerGrpcSpec](#schedulergrpcspec) + +### `cache_lookup` + +**Type:** [CacheLookupSpec](#cachelookupspec) + +### `property_modifier` + +**Type:** [PropertyModifierSpec](#propertymodifierspec) + +### `historical_resource` + +**Type:** [HistoricalResourceSpec](#historicalresourcespec) + +## ServerConfig + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `name` | string | — | {index of server in config} | Name of the server. This is used to help identify the service for telemetry and logs. | +| `listener` | [ListenerConfig](#listenerconfig) | Yes | — | Configuration | +| `services` | [ServicesConfig](#servicesconfig) | — | — | Services to attach to server. | +| `experimental_identity_header` | [IdentityHeaderSpec](#identityheaderspec) | — | {see `IdentityHeaderSpec`} | The config related to identifying the client. | + +## OriginEventsSpec + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `publisher` | [OriginEventsPublisherSpec](#origineventspublisherspec) | Yes | — | The publisher configuration for origin events. | +| `max_event_queue_size` | integer (uint) | — | 65536 (zero defaults to this) | The maximum number of events to queue before applying back pressure. IMPORTANT: Backpressure causes all clients to slow down significantly. Zero is default. | + +## GlobalConfig + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `max_open_files` | integer (uint) | Yes | 24576 (= 24 * 1024) | Maximum number of open files that can be opened at one time. This value is not strictly enforced, it is a best effort. Some internal libraries open files or read metadata from a files which do not obey this limit, however the vast majority of cases will have this limit be honored. This value must be larger than `ulimit -n` to have any effect. Any network open file descriptors is not counted in this limit, but is counted in the kernel limit. It is a good idea to set a very large `ulimit -n`. Note: This value must be greater than 10. | +| `default_digest_hash_function` | [ConfigDigestHashFunction](#configdigesthashfunction) | — | `ConfigDigestHashFunction::sha256` | Default hash function to use while uploading blobs to the CAS when not set by client. | +| `default_digest_size_health_check` | integer (uint) | — | 1024*1024 (1MiB) | Default digest size to use for health check when running diagnostics checks. Health checks are expected to use this size for filling a buffer that is used for creation of digest. | + +## CacheMetricsSpec + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `cache_type` | string | Yes | — | Low-cardinality cache type label for metrics, for example `cas` or `ac`. | +| `backend` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | Store to wrap with cache operation metrics. | + +## MemorySpec + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `eviction_policy` | [EvictionPolicy](#evictionpolicy) | — | — | Policy used to evict items out of the store. Failure to set this value will cause items to never be removed from the store causing infinite memory usage. | + +## ExperimentalCloudObjectSpec + +See [`experimental_cloud_object_store`](#experimental_cloud_object_store-1) for details + +## OntapS3ExistenceCacheSpec + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `index_path` | string | Yes | — | | +| `sync_interval_seconds` | integer (uint32) | Yes | — | | +| `backend` | [ExperimentalOntapS3Spec](#experimentalontaps3spec) | Yes | — | | + +## VerifySpec + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `backend` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | The underlying store wrap around. All content will first flow through self before forwarding to backend. In the event there is an error detected in self, the connection to the backend will be terminated, and early termination should always cause updates to fail on the backend. | +| `verify_size` | boolean | — | `false` | If set the store will verify the size of the data before accepting an upload of data. | +| `verify_hash` | boolean | — | `false` | If the data should be hashed and verify that the key matches the computed hash. The hash function is automatically determined based request and if not set will use the global default. | + +## CompletenessCheckingSpec + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `backend` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | The underlying store that will have it's results validated before sending to client. | +| `cas_store` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | When a request is made, the results are decoded and all output digests/files are verified to exist in this CAS store before returning success. | + +## CompressionSpec + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `backend` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | The underlying store wrap around. All content will first flow through self before forwarding to backend. In the event there is an error detected in self, the connection to the backend will be terminated, and early termination should always cause updates to fail on the backend. | +| `compression_algorithm` | [CompressionAlgorithm](#compressionalgorithm) | Yes | — | The compression algorithm to use. | + +## DedupSpec + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `index_store` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | Store used to store the index of each dedup slice. This store should generally be fast and small. | +| `content_store` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | The store where the individual chunks will be uploaded. This store should generally be the slower & larger store. | +| `min_size` | integer (uint32) | — | 64k | Minimum size that a chunk will be when slicing up the content. Note: This setting can be increased to improve performance because it will actually not check this number of bytes when deciding where to partition the data. | +| `normal_size` | integer (uint32) | — | 256k | A best-effort attempt will be made to keep the average size of the chunks to this number. It is not a guarantee, but a slight attempt will be made. | +| `max_size` | integer (uint32) | — | 512k | Maximum size a chunk is allowed to be. | +| `max_concurrent_fetch_per_get` | integer (uint32) | — | 10 | Due to implementation detail, we want to prefer to download the first chunks of the file so we can stream the content out and free up some of our buffers. This configuration will be used to restrict the number of concurrent chunk downloads at a time per `get()` request. | + +## ExistenceCacheSpec + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `backend` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | The underlying store wrap around. All content will first flow through self before forwarding to backend. In the event there is an error detected in self, the connection to the backend will be terminated, and early termination should always cause updates to fail on the backend. | +| `eviction_policy` | [EvictionPolicy](#evictionpolicy) | — | — | Policy used to evict items out of the store. Failure to set this value will cause items to never be removed from the store causing infinite memory usage. | + +## FastSlowSpec + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `fast` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | Fast store that will be attempted to be contacted before reaching out to the `slow` store. | +| `fast_direction` | [StoreDirection](#storedirection) | — | `"both"` | How to handle the fast store. This can be useful to set to Get for worker nodes such that results are persisted to the slow store only. | +| `slow` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | If the object does not exist in the `fast` store it will try to get it from this store. | +| `slow_direction` | [StoreDirection](#storedirection) | — | `"both"` | How to handle the slow store. This can be useful if creating a diode and you wish to have an upstream read only store. | +| `bypass_dedup_threshold_bytes` | integer (uint64) | — | disabled (0) | Reads of blobs at or above this size skip the leader/follower dedup map and stream straight from the slow store without populating the fast tier. `0` (the default) disables the bypass: every read goes through dedup, matching the prior behaviour. Enable it by setting a threshold — 256 MiB is a reasonable starting point for backends where large-blob dedup is a net loss (followers tend to time out anyway), but the right value is workload-dependent. | + +## ShardSpec + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `stores` | array of [ShardConfig](#shardconfig) | Yes | — | Stores to shard the data to. | + +## FilesystemSpec + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `content_path` | string | Yes | — | Path on the system where to store the actual content. This is where the bulk of the data will be placed. On service startup this folder will be scanned and all files will be added to the cache. In the event one of the files doesn't match the criteria, the file will be deleted. | +| `temp_path` | string | Yes | — | A temporary location of where files that are being uploaded or deleted will be placed while the content cannot be guaranteed to be accurate. This location must be on the same block device as `content_path` so atomic moves can happen (i.e., move without copy). All files in this folder will be deleted on every startup. | +| `read_buffer_size` | integer (uint32) | — | 32k | Buffer size to use when reading files. Generally this should be left to the default value except for testing. | +| `eviction_policy` | [EvictionPolicy](#evictionpolicy) | — | — | Policy used to evict items out of the store. Failure to set this value will cause items to never be removed from the store causing infinite memory usage. | +| `block_size` | integer (uint64) | — | 4kb | The block size of the filesystem for the running machine value is used to determine an entry's actual size on disk consumed For a 4KB block size filesystem, a 1B file actually consumes 4KB | +| `max_concurrent_writes` | integer (uint) | — | unlimited | Maximum number of concurrent write operations allowed. Each write involves streaming data to a temp file and calling `sync_all()`, which can saturate disk I/O when many writes happen simultaneously. Limiting concurrency prevents disk saturation from blocking the async runtime. A value of 0 means unlimited (no concurrency limit). | + +## RefSpec + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `name` | string | Yes | — | Name of the store under the root "stores" config object. | + +## SizePartitioningSpec + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `size` | integer (uint64) | Yes | — | Size to partition the data on. | +| `lower_store` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | Store to send data when object is < (less than) size. | +| `upper_store` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | Store to send data when object is >= (less than eq) size. | + +## GrpcSpec + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `instance_name` | string | — | `""` | Instance name for GRPC calls. Proxy calls will have the `instance_name` changed to this. | +| `endpoints` | array of [GrpcEndpoint](#grpcendpoint) | Yes | — | The endpoint of the grpc connection. | +| `store_type` | [StoreType](#storetype) | Yes | — | The type of the upstream store, this ensures that the correct server calls are made. | +| `retry` | [Retry](#retry) | — | — | Retry configuration to use when a network request fails. | +| `max_concurrent_requests` | integer (uint) | — | `0` | Limit the number of simultaneous upstream requests to this many. A value of zero is treated as unlimited. If the limit is reached the request is queued. | +| `connections_per_endpoint` | integer (uint) | — | `0` | The number of connections to make to each specified endpoint to balance the load over multiple TCP connections. Default 1. | +| `rpc_timeout_s` | integer (uint64) | — | 0 (disabled) | Maximum time (seconds) allowed for a single RPC request (e.g. a `ByteStream.Write` call) before it is cancelled. | +| `use_legacy_resource_names` | boolean | — | false | Use legacy `ByteStream` resource name format, omitting the digest function component from the path. | +| `headers` | map of string to string | — | — | Static headers to attach to every outgoing gRPC request sent to this store's upstream endpoints. Useful for fixed authentication tokens (e.g. `{"authorization": "Bearer "}`) and other static metadata. | +| `forward_headers` | array of string | — | — | Header names to forward from the incoming client request to every outgoing upstream request. The header value is taken from the client request that triggered this store operation. Use this to pass through dynamic credentials such as JWT tokens sent by build clients. | + +## RedisSpec + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `addresses` | array of string | Yes | — | The hostname or IP address of the Redis server. Ex: `["redis://username:password@redis-server-url:6380/99"]` 99 Represents database ID, 6380 represents the port. | +| `response_timeout_s` | integer (uint64) | — | 10 | DEPRECATED: use `command_timeout_ms` The response timeout for the Redis connection in seconds. | +| `connection_timeout_s` | integer (uint64) | — | 10 | DEPRECATED: use `connection_timeout_ms` | +| `experimental_pub_sub_channel` | string | — | (Empty String / No Channel) | An optional and experimental Redis channel to publish write events to. | +| `key_prefix` | string | — | (Empty String / No Prefix) | An optional prefix to prepend to all keys in this store. | +| `mode` | [RedisMode](#redismode) | — | standard, | Set the mode Redis is operating in. | +| `broadcast_channel_capacity` | integer (uint) | — | `0` | Deprecated as redis-rs doesn't use it | +| `command_timeout_ms` | integer (uint64) | — | 10000 (10 seconds) | The amount of time in milliseconds until the Redis store considers the command to be timed out. This will trigger a retry of the command and potentially a reconnection to the Redis server. | +| `connection_timeout_ms` | integer (uint64) | — | 3000 (3 seconds) | The amount of time in milliseconds until the Redis store considers the connection to unresponsive. This will trigger a reconnection to the Redis server. | +| `health_check_timeout_ms` | integer (uint64) | — | 4000 (4 seconds) | Per-call ceiling for the `check_health` PING in milliseconds. | +| `read_chunk_size` | integer (uint) | — | 64KiB | The amount of data to read from the Redis server at a time. This is used to limit the amount of memory used when reading large objects from the Redis server as well as limiting the amount of time a single read operation can take. | +| `connection_pool_size` | integer (uint) | — | 3 | The number of connections to keep open to the Redis servers. | +| `max_chunk_uploads_per_update` | integer (uint) | — | 10 | The maximum number of upload chunks to allow per update. This is used to limit the amount of memory used when uploading large objects to the Redis server. A good rule of thumb is to think of the data as: `AVAIL_MEMORY / (read_chunk_size * max_chunk_uploads_per_update) = THORETICAL_MAX_CONCURRENT_UPLOADS` (note: it is a good idea to divide `AVAIL_MAX_MEMORY` by ~10 to account for other memory usage) | +| `scan_count` | integer (uint) | — | 10000 | The COUNT value passed when scanning keys in Redis. This is used to hint the amount of work that should be done per response. | +| `retry` | [Retry](#retry) | — | — | Retry configuration to use when a network request fails. | +| `max_client_permits` | integer (uint) | — | 500 | Maximum number of permitted actions to the Redis store at any one time This stops problems with timeouts due to many, many inflight actions | +| `max_count_per_cursor` | integer (uint64) | — | 1500 | Maximum number of items returned per cursor for the search indexes May reduce thundering herd issues with worker provisioner at higher node counts, | + +## NoopSpec + +_No fields._ + +## ExperimentalMongoSpec + +Configuration for `ExperimentalMongoDB` store. + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `connection_string` | string | Yes | — | `ExperimentalMongoDB` connection string. Example: <mongodb://localhost:27017> or <mongodb+srv://cluster.mongodb.net> | +| `database` | string | — | "nativelink" | The database name to use. | +| `cas_collection` | string | — | "cas" | The collection name for CAS data. | +| `scheduler_collection` | string | — | "scheduler" | The collection name for scheduler data. | +| `key_prefix` | string | — | "" | Prefix to prepend to all keys stored in `MongoDB`. | +| `read_chunk_size` | integer (uint) | — | 65536 (64KB) | The maximum amount of data to read from `MongoDB` in a single chunk (in bytes). | +| `max_concurrent_uploads` | integer (uint) | — | 10 | Deprecated, unused Maximum number of concurrent uploads allowed. | +| `connection_timeout_ms` | integer (uint64) | — | 3000 | Connection timeout in milliseconds. | +| `command_timeout_ms` | integer (uint64) | — | 10000 | Command timeout in milliseconds. | +| `enable_change_streams` | boolean | — | false | Enable `MongoDB` change streams for real-time updates. Required for scheduler subscriptions. | +| `write_concern_w` | string | — | — | Write concern 'w' parameter. Can be a number (e.g., 1) or string (e.g., "majority"). | +| `write_concern_j` | boolean | — | — | Write concern 'j' parameter (journal acknowledgment). | +| `write_concern_timeout_ms` | integer (uint32) | — | — | Write concern timeout in milliseconds. | +| `max_requests` | integer (uint) | — | Unlimited | Limits the number of requests at any one time | + +## LocalWorkerConfig + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `name` | string | — | {Index position in the workers list} | Name of the worker. This is give a more friendly name to a worker for logging and metric publishing. This is also the prefix of the worker id (i.e., "{name}{uuidv6}"). | +| `worker_api_endpoint` | [EndpointConfig](#endpointconfig) | Yes | — | Endpoint which the worker will connect to the scheduler's `WorkerApiService`. | +| `max_action_timeout_s` | integer (uint) | — | 20 minutes | The maximum time an action is allowed to run. If a task requests for a timeout longer than this time limit, the task will be rejected. Value in seconds. | +| `max_upload_timeout_s` | integer (uint) | — | 10 minutes | Maximum time allowed for uploading action results to CAS after execution completes. If upload takes longer than this, the action fails with `DeadlineExceeded` and may be retried by the scheduler. Value in seconds. | +| `max_cleanup_wait_s` | integer (uint) | — | 30 seconds | Maximum time to wait for action directory cleanup before timing out. Value in seconds. | +| `max_cleanup_backoff_ms` | integer (uint) | — | 500 milliseconds | Maximum backoff duration for exponential backoff when waiting for cleanup. Value in milliseconds. | +| `max_inflight_tasks` | integer (uint64) | — | 0 (infinite tasks) | Maximum number of inflight tasks this worker can cope with. | +| `timeout_handled_externally` | boolean | — | false (`NativeLink` fully handles timeouts) | If timeout is handled in `entrypoint` or another wrapper script. If set to true `NativeLink` will not honor the timeout the action requested and instead will always force kill the action after `max_action_timeout` has been reached. If this is set to false, the smaller value of the action's timeout and `max_action_timeout` will be used to which `NativeLink` will kill the action. | +| `entrypoint` | string | — | {Use the command from the job request} | The command to execute on every execution request. This will be parsed as a command + arguments (not shell). Example: "run.sh" and a job with command: "sleep 5" will result in a command like: "run.sh sleep 5". | +| `experimental_precondition_script` | string | — | — | An optional script to run before every action is processed on the worker. The value should be the full path to the script to execute and will pause all actions on the worker if it returns an exit code other than 0. If not set, then the worker will never pause and will continue to accept jobs according to the scheduler configuration. This is useful, for example, if the worker should not take any more actions until there is enough resource available on the machine to handle them. | +| `cas_fast_slow_store` | string | Yes | — | Underlying CAS store that the worker will use to download CAS artifacts. This store must be a `FastSlowStore`. The `fast` store must be a `FileSystemStore` because it will use hardlinks when building out the files instead of copying the files. The slow store must eventually resolve to the same store the scheduler/client uses to send job requests. | +| `upload_action_result` | [UploadActionResultConfig](#uploadactionresultconfig) | — | — | Configuration for uploading action results. | +| `work_directory` | string | Yes | — | The directory work jobs will be executed from. This directory will be fully managed by the worker service and will be purged on startup. This directory and the directory referenced in `local_filesystem_store_ref`'s `stores::FilesystemStore::content_path` must be on the same filesystem. Hardlinks will be used when placing files that are accessible to the jobs that are sourced from `local_filesystem_store_ref`'s `content_path`. | +| `platform_properties` | map of string to [WorkerProperty](#workerproperty) | Yes | — | Properties of this worker. This configuration will be sent to the scheduler and used to tell the scheduler to restrict what should be executed on this worker. | +| `additional_environment` | map of string to [EnvironmentSource](#environmentsource) | — | — | An optional mapping of environment names to set for the execution as well as those specified in the action itself. If set, will set each key as an environment variable before executing the job with the value of the environment variable being the value of the property of the action being executed of that name or the fixed value. | +| `directory_cache` | [DirectoryCacheConfig](#directorycacheconfig) | — | — | Optional directory cache configuration for improving performance by caching reconstructed input directories and using hardlinks instead of rebuilding them from CAS for every action. | +| `use_namespaces` | boolean | — | False | Whether to use namespaces to isolate the execution. This is only available on Linux. It is highly recommended as it avoids a number of issues with zombie processes and also provides additional hermeticity. If explicitly set to true and it is not supported the worker will exit with an error. | +| `use_mount_namespace` | boolean | — | False | Whether to use a mount namespace to isolate the worker root. This is only available on Linux and when `use_namespaces` is true. It is highly recommended provides additional hermeticity. If explicitly set to true and it is not supported or `use_namespaces` is not set to true the worker will exit with an error. | + +## SimpleSpec + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `supported_platform_properties` | map of string to [PropertyType](#propertytype) | — | — | A list of supported platform properties mapped to how these properties are used when the scheduler looks for worker nodes capable of running the task. | +| `retain_completed_for_s` | integer (uint32) | — | 60 seconds | The amount of time to retain completed actions for in case a `WaitExecution` is called after the action has completed. | +| `client_action_timeout_s` | integer (uint64) | — | 60 seconds | Mark operations as completed with error if no client has updated them within this duration. | +| `worker_timeout_s` | integer (uint64) | — | 5 seconds | Remove workers from pool once the worker has not responded in this amount of time in seconds. | +| `max_action_executing_timeout_s` | integer (uint64) | — | 0 (disabled) | Maximum time (seconds) an action can stay in Executing state without any worker update before being timed out and re-queued. This applies regardless of worker keepalive status, catching cases where a worker is alive (sending keepalives) but stuck on a specific action. Set to 0 to disable (relies only on `worker_timeout_s`). | +| `max_job_retries` | integer (uint) | — | 3 | If a job returns an internal error or times out this many times when attempting to run on a worker the scheduler will return the last error to the client. Jobs will be retried and this configuration is to help prevent one rogue job from infinitely retrying and taking up a lot of resources when the task itself is the one causing the server to go into a bad state. | +| `allocation_strategy` | [WorkerAllocationStrategy](#workerallocationstrategy) | — | `"least_recently_used"` | The strategy used to assign workers jobs. | +| `experimental_backend` | [ExperimentalSimpleSchedulerBackend](#experimentalsimpleschedulerbackend) | — | memory | The storage backend to use for the scheduler. | +| `worker_match_logging_interval_s` | integer (int64) | — | `10` | Every N seconds, do logging of worker matching e.g. "worker busy", "can't find any worker" Defaults to 10s. Can be set to `-1` to disable | + +## SchedulerGrpcSpec + +A scheduler that forwards requests to an upstream scheduler. This +is useful to use when doing some kind of local action cache or CAS away from +the main cluster of workers. In general, it's more efficient to point the +build at the main scheduler directly though. + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `endpoint` | [GrpcEndpoint](#grpcendpoint) | Yes | — | The upstream scheduler to forward requests to. | +| `retry` | [Retry](#retry) | — | — | Retry configuration to use when a network request fails. | +| `max_concurrent_requests` | integer (uint) | — | unlimited | Limit the number of simultaneous upstream requests to this many. A value of zero is treated as unlimited. If the limit is reached the request is queued. | +| `connections_per_endpoint` | integer (uint) | — | 1 | The number of connections to make to each specified endpoint to balance the load over multiple TCP connections. | + +## CacheLookupSpec + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `ac_store` | string | Yes | — | The reference to the action cache store used to return cached actions from rather than running them again. To prevent unintended issues, this store should probably be a `CompletenessCheckingSpec`. | +| `scheduler` | [SchedulerSpec](#schedulerspec) | Yes | — | The nested scheduler to use if cache lookup fails. | + +## PropertyModifierSpec + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `modifications` | array of [PropertyModification](#propertymodification) | Yes | — | A list of modifications to perform to incoming actions for the nested scheduler. These are performed in order and blindly, so removing a property that doesn't exist is fine and overwriting an existing property is also fine. If adding properties that do not exist in the nested scheduler is not supported and will likely cause unexpected behaviour. | +| `scheduler` | [SchedulerSpec](#schedulerspec) | Yes | — | The nested scheduler to use after modifying the properties. | + +## HistoricalResourceSpec + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `hints_file` | string | Yes | — | JSON file containing historical resource hints keyed by Bazel `RequestMetadata` `target_id` and/or `action_mnemonic`. | +| `refresh_interval_s` | integer (uint64) | — | 30 seconds | Reload interval for `hints_file`. Set to 0 to load once. | +| `cpu_property_name` | string | — | `cpu_count` | Platform property name used for CPU minimum values. | +| `memory_property_name` | string | — | `memory_kb` | Platform property name used for memory minimum values, expressed in KiB. | +| `scheduler` | [SchedulerSpec](#schedulerspec) | Yes | — | The nested scheduler to use after applying resource hints. | + +## ListenerConfig + +Plus exactly one of the following variants (the key selects the variant): + +### `http` + +Listener for HTTP/HTTPS/HTTP2 sockets. + +**Type:** [HttpListener](#httplistener) + +## ServicesConfig + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `cas` | array of [CasServiceConfig](#casserviceconfig) | — | — | The Content Addressable Storage (CAS) backend config. The key is the `instance_name` used in the protocol and the value is the underlying CAS store config. | +| `ac` | array of [ActionCacheServiceConfig](#actioncacheserviceconfig) | — | — | The Action Cache (AC) backend config. The key is the `instance_name` used in the protocol and the value is the underlying AC store config. | +| `capabilities` | array of [CapabilitiesServiceConfig](#capabilitiesserviceconfig) | — | — | Capabilities service is required in order to use most of the bazel protocol. This service is used to provide the supported features and versions of this bazel GRPC service. | +| `execution` | array of [ExecutionServiceConfig](#executionserviceconfig) | — | — | The remote execution service configuration. NOTE: This service is under development and is currently just a place holder. | +| `bytestream` | array of [ByteStreamServiceConfig](#bytestreamserviceconfig) | — | — | This is the service used to stream data to and from the CAS. Bazel's protocol strongly encourages users to use this streaming interface to interact with the CAS when the data is large. | +| `fetch` | array of [FetchServiceConfig](#fetchserviceconfig) | — | — | These two are collectively the Remote Asset protocol, but it's defined as two separate services | +| `push` | array of [PushServiceConfig](#pushserviceconfig) | — | — | | +| `worker_api` | [WorkerApiConfig](#workerapiconfig) | — | — | This is the service used for workers to connect and communicate through. NOTE: This service should be served on a different, non-public port. In other words, `worker_api` configuration should not have any other services that are served on the same port. Doing so is a security risk, as workers have a different permission set than a client that makes the remote execution/cache requests. | +| `experimental_bep` | [BepConfig](#bepconfig) | — | — | Experimental - Build Event Protocol (BEP) configuration. This is the service that will consume build events from the client and publish them to a store for processing by an external service. | +| `admin` | [AdminConfig](#adminconfig) | — | — | This is the service for any administrative tasks. It provides a REST API endpoint for administrative purposes. | +| `health` | [HealthConfig](#healthconfig) | — | — | This is the service for health status check. | + +## IdentityHeaderSpec + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `header_name` | string | — | "x-identity" | The name of the header to look for the identity in. | +| `required` | boolean | — | `false` | If the header is required to be set or fail the request. | + +## OriginEventsPublisherSpec + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `store` | string | Yes | — | The store to publish nativelink events to. The store name referenced in the `stores` map in the main config. | + +## ConfigDigestHashFunction + +| Value | Description | +| --- | --- | +| `"sha256"` | Use the sha256 hash function. [https://en.wikipedia.org/wiki/SHA-2](https://en.wikipedia.org/wiki/SHA-2) | +| `"blake3"` | Use the blake3 hash function. [https://en.wikipedia.org/wiki/BLAKE_(hash_function)](https://en.wikipedia.org/wiki/BLAKE_(hash_function)) | + +## EvictionPolicy + +Eviction policy always works on LRU (Least Recently Used). Any time an entry +is touched it updates the timestamp. Inserts and updates will execute the +eviction policy removing any expired entries and/or the oldest entries +until the store size becomes smaller than `max_bytes`. + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `max_bytes` | integer (uint) | — | 0 | Maximum number of bytes before eviction takes place. Zero means never evict based on size. | +| `evict_bytes` | integer (uint) | — | 0 | When eviction starts based on hitting `max_bytes`, continue until `max_bytes - evict_bytes` is met to create a low watermark. This stops operations from thrashing when the store is close to the limit. | +| `max_seconds` | integer (uint32) | — | 0 | Maximum number of seconds for an entry to live since it was last accessed before it is evicted. Zero means never evict based on time. | +| `max_count` | integer (uint64) | — | 0 | Maximum size of the store before an eviction takes place. Zero means never evict based on count. | + +## Retry + +Retry configuration. This configuration is exponential and each iteration +a jitter as a percentage is applied of the calculated delay. For example: +```haskell +Retry{ + max_retries: 7, + delay: 0.1, + jitter: 0.5, +} +``` +will result in: + +| Attempt | Delay | +| --- | --- | +| 1 | 0 ms | +| 2 | 75 to 125 ms | +| 3 | 150 to 250 ms | +| 4 | 300 to 500 ms | +| 5 | 600 ms to 1 s | +| 6 | 1.2 to 2 s | +| 7 | 2.4 to 4 s | +| 8 | 4.8 to 8 s | + +The total delay is additive, so this example produces 9.525 to 15.875 s of total delay for a single request. + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `max_retries` | integer (uint) | — | `0` | Maximum number of retries until retrying stops. Setting this to zero will always attempt 1 time, but not retry. | +| `delay` | number (float) | — | `0` | Delay in seconds for exponential back off. | +| `jitter` | number (float) | — | `0` | Amount of jitter to add as a percentage in decimal form. This will change the formula like: | +| `retry_on_errors` | array of [ErrorCode](#errorcode) | — | — | A list of error codes to retry on, if this is not set then the default error codes to retry on are used. These default codes are the most likely to be non-permanent: `Unknown`; `Cancelled`; `DeadlineExceeded`; `ResourceExhausted`; `Aborted`; `Internal`; `Unavailable`; `DataLoss` | + +## ExperimentalOntapS3Spec + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `endpoint` | string | Yes | — | | +| `vserver_name` | string | Yes | — | | +| `bucket` | string | Yes | — | | +| `root_certificates` | string | — | — | | +| `key_prefix` | string | — | — | If you wish to prefix the location in the bucket. If None, no prefix will be used. | +| `retry` | [Retry](#retry) | — | — | Retry configuration to use when a network request fails. | +| `consider_expired_after_s` | integer (uint32) | — | 0 | If the number of seconds since the `last_modified` time of the object is greater than this value, the object will not be considered "existing". This allows for external tools to delete objects that have not been uploaded in a long time. If a client receives a `NotFound` the client should re-upload the object. | +| `max_retry_buffer_per_request` | integer (uint) | — | 5MB | The maximum buffer size to retain in case of a retryable error during upload. Setting this to zero will disable upload buffering; this means that in the event of a failure during upload, the entire upload will be aborted and the client will likely receive an error. | +| `multipart_max_concurrent_uploads` | integer (uint) | — | 10 | Maximum number of concurrent `UploadPart` requests per `MultipartUpload`. | +| `insecure_allow_http` | boolean | — | false | Allow unencrypted HTTP connections. Only use this for local testing. | +| `disable_http2` | boolean | — | false | Disable HTTP/2 connections and only use HTTP/1.1. Default client configuration will have HTTP/1.1 and HTTP/2 enabled for connection schemes. HTTP/2 should be disabled if environments have poor support or performance related to HTTP/2. Safe to keep default unless underlying network environment, S3, or GCS API servers specify otherwise. | + +## CompressionAlgorithm + +Plus exactly one of the following variants (the key selects the variant): + +### `lz4` + +LZ4 compression algorithm is extremely fast for compression and +decompression, however does not perform very well in compression +ratio. In most cases build artifacts are highly compressible, however +lz4 is quite good at aborting early if the data is not deemed very +compressible. + +see: [https://lz4.github.io/lz4/](https://lz4.github.io/lz4/) + +**Type:** [Lz4Config](#lz4config) + +## StoreDirection + +| Value | Description | +| --- | --- | +| `"both"` | The store operates normally and all get and put operations are handled by it. | +| `"update"` | Update operations will cause persistence to this store, but Get operations will be ignored. This only makes sense on the fast store as the slow store will never get written to on Get anyway. | +| `"get"` | Get operations will cause persistence to this store, but Update operations will be ignored. | +| `"read_only"` | Operate as a read only store, only really makes sense if there's another way to write to it. | + +## ShardConfig + +Configuration for an individual shard of the store. + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `store` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | Store to shard the data to. | +| `weight` | integer (uint32) | — | 1 | The weight of the store. This is used to determine how much data should be sent to the store. The actual percentage is the sum of all the store's weights divided by the individual store's weight. | + +## GrpcEndpoint + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `address` | string | Yes | — | The endpoint address (i.e. grpc://example.com:443 or grpcs://example.com:443). | +| `tls_config` | [ClientTlsConfig](#clienttlsconfig) | — | — | The TLS configuration to use to connect to the endpoint (if grpcs). | +| `concurrency_limit` | integer (uint) | — | — | The maximum concurrency to allow on this endpoint. | +| `connect_timeout_s` | integer (uint64) | — | 30 seconds | Timeout for establishing a TCP connection to the endpoint (seconds). | +| `tcp_keepalive_s` | integer (uint64) | — | 30 seconds | TCP keepalive interval (seconds). Sends TCP keepalive probes at this interval to detect dead connections at the OS level. | +| `http2_keepalive_interval_s` | integer (uint64) | — | 30 seconds | HTTP/2 keepalive interval (seconds). Sends HTTP/2 PING frames at this interval to detect dead connections at the application level. | +| `http2_keepalive_timeout_s` | integer (uint64) | — | 20 seconds | HTTP/2 keepalive timeout (seconds). If a PING response is not received within this duration, the connection is considered dead. | + +## StoreType + +| Value | Description | +| --- | --- | +| `"cas"` | The store is content addressable storage. | +| `"ac"` | The store is an action cache. | + +## RedisMode + +| Value | Description | +| --- | --- | +| `"cluster"` | Use Redis Cluster. | +| `"sentinel"` | Use Redis Sentinel. | +| `"standard"` | Use a standalone Redis server. | + +## EndpointConfig + +Generic config for an endpoint and associated configs. + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `uri` | string | Yes | — | URI of the endpoint. | +| `timeout` | number (float) | — | 5 seconds | Timeout in seconds that a request should take. | +| `tls_config` | [ClientTlsConfig](#clienttlsconfig) | — | — | The TLS configuration to use to connect to the endpoint. | + +## UploadActionResultConfig + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `ac_store` | string | — | {No uploading is done} | Underlying AC store that the worker will use to publish execution results into. Objects placed in this store should be reachable from the scheduler/client-cas after they have finished updating. | +| `upload_ac_results_strategy` | [UploadCacheResultsStrategy](#uploadcacheresultsstrategy) | — | `SuccessOnly` | In which situations should the results be published to the `ac_store`, if set to `SuccessOnly` then only results with an exit code of 0 will be uploaded, if set to Everything all completed results will be uploaded. | +| `historical_results_store` | string | — | {CAS store of parent} | Store to upload historical results to. This should be a CAS store if set. | +| `upload_historical_results_strategy` | [UploadCacheResultsStrategy](#uploadcacheresultsstrategy) | — | `FailuresOnly` | In which situations should the results be published to the historical CAS. The historical CAS is where failures are published. These messages conform to the CAS key-value lookup format and are always a `HistoricalExecuteResponse` serialized message. | +| `success_message_template` | string | — | "" (no message) | Template to use for the `ExecuteResponse.message` property. This message is attached to the response before it is sent to the client. The following special variables are supported:; `digest_function`: Digest function used to calculate the action digest: `action_digest_hash`: Action digest hash: `action_digest_size`: Action digest size: `historical_results_hash`: `HistoricalExecuteResponse` digest hash: `historical_results_size`: `HistoricalExecuteResponse` digest size. | +| `failure_message_template` | string | — | "" (no message) | Same as `success_message_template` but for failure case. | + +## WorkerProperty + +Plus exactly one of the following variants (the key selects the variant): + +### `values` + +List of static values. +Note: Generally there should only ever be 1 value, but if the platform +property key is `PropertyType::Priority` it may have more than one value. + +**Type:** array of string + +### `query_cmd` + +A dynamic configuration. The string will be executed as a command +(not shell) and will be split by "\n" (new line character). + +**Type:** string + +## EnvironmentSource + +One of: + +- `property`: string — The name of the platform property in the action to get the value from. +- `value`: string — The raw value to set. +- `"from_environment"` — Take the value from the local environment corresponding to the name key +- `"timeout_millis"` — The max amount of time in milliseconds the command is allowed to run (requested by the client). +- `"side_channel_file"` — A special file path will be provided that can be used to communicate with the parent process about out-of-band information. This file will be read after the command has finished executing. Based on the contents of the file, the behavior of the result may be modified. +- `"action_directory"` — A "root" directory for the action. This directory can be used to store temporary files that are not needed after the action has completed. This directory will be purged after the action has completed. + +## DirectoryCacheConfig + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `max_entries` | integer (uint) | — | 1000 | Maximum number of cached directories. | +| `max_size_bytes` | integer (uint64) | — | 10737418240 (10 GB) | Maximum total size in bytes for all cached directories (0 = unlimited). | +| `cache_root` | string | — | `{work_directory}/../directory_cache` | Base directory for cache storage. This directory will be managed by the worker and should be on the same filesystem as `work_directory`. | + +## PropertyType + +When the scheduler matches tasks to workers that are capable of running +the task, this value will be used to determine how the property is treated. + +One of: + +- string +- `"minimum"` — Requires the platform property to be a u64 and when the scheduler looks for appropriate worker nodes that are capable of executing the task, the task will not run on a node that has less than this value. +- `"exact"` — Requires the platform property to be a string and when the scheduler looks for appropriate worker nodes that are capable of executing the task, the task will not run on a node that does not have this property set to the value with exact string match. +- `"priority"` — Does not restrict on this value and instead will be passed to the worker as an informational piece. In the future this will be used by the scheduler and worker to cause the scheduler to prefer certain workers over others, but not restrict them based on these values. + +## WorkerAllocationStrategy + +When a worker is being searched for to run a job, this will be used +on how to choose which worker should run the job when multiple +workers are able to run the task. + +| Value | Description | +| --- | --- | +| `"least_recently_used"` | Prefer workers that have been least recently used to run a job. | +| `"most_recently_used"` | Prefer workers that have been most recently used to run a job. | + +## ExperimentalSimpleSchedulerBackend + +One of: + +- `"memory"` — Use an in-memory store for the scheduler. +- `redis`: [ExperimentalRedisSchedulerBackend](#experimentalredisschedulerbackend) — Use a Redis store for the scheduler. + +## SchedulerSpec + +Plus exactly one of the following variants (the key selects the variant): + +### `simple` + +**Type:** [SimpleSpec](#simplespec) + +### `grpc` + +**Type:** [SchedulerGrpcSpec](#schedulergrpcspec) + +### `cache_lookup` + +**Type:** [CacheLookupSpec](#cachelookupspec) + +### `property_modifier` + +**Type:** [PropertyModifierSpec](#propertymodifierspec) + +### `historical_resource` + +**Type:** [HistoricalResourceSpec](#historicalresourcespec) + +## PropertyModification + +Plus exactly one of the following variants (the key selects the variant): + +### `add` + +Add a property to the action properties. + +**Type:** [PlatformPropertyAddition](#platformpropertyaddition) + +### `remove` + +Remove a named property from the action. + +**Type:** string + +### `replace` + +If a property is found, then replace it with another one. + +**Type:** [PlatformPropertyReplacement](#platformpropertyreplacement) + +## HttpListener + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `socket_address` | string | Yes | — | Address to listen on. Example: `127.0.0.1:8080` or `:8080` to listen to all IPs. | +| `freebind` | boolean | — | false | Allow binding `socket_address` before it is assigned locally. | +| `compression` | [HttpCompressionConfig](#httpcompressionconfig) | — | — | Data transport compression configuration to use for this service. | +| `advanced_http` | [HttpServerConfig](#httpserverconfig) | — | — | Advanced HTTP server configuration. | +| `max_decoding_message_size` | integer (uint) | — | 4 MiB | Maximum number of bytes to decode on each grpc stream chunk. | +| `tls` | [TlsConfig](#tlsconfig) | — | — | TLS configuration for this server. If not set, the server will not use TLS. | + +## CasServiceConfig + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `instance_name` | string | — | `""` | Used when the config references `instance_name` in the protocol. | +| `cas_store` | string | Yes | — | The store name referenced in the `stores` map in the main config. This store name referenced here may be reused multiple times. | +| `experimental_chunking` | [CasChunkingConfig](#caschunkingconfig) | — | not set — chunking RPCs are rejected, nothing is advertised, | Optional and experimental: enables the REAPI `SplitBlob`/`SpliceBlob` RPCs used by content-defined chunking clients (e.g. Bazel's `--experimental_remote_cache_chunking`). When set, the capabilities service advertises blob split/splice support and `FastCDC` 2020 parameters for this instance. When `cas_store` is a grpc store the RPCs are forwarded to the backend (which must support chunking with matching parameters); otherwise they are served locally. | + +## ActionCacheServiceConfig + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `instance_name` | string | — | `""` | Used when the config references `instance_name` in the protocol. | +| `ac_store` | string | Yes | — | The store name referenced in the `stores` map in the main config. This store name referenced here may be reused multiple times. | +| `read_only` | boolean | — | `false` | Whether the Action Cache store may be written to, this if set to false it is only possible to read from the Action Cache. | + +## CapabilitiesServiceConfig + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `instance_name` | string | — | `""` | Used when the config references `instance_name` in the protocol. | +| `remote_execution` | [CapabilitiesRemoteExecutionConfig](#capabilitiesremoteexecutionconfig) | — | — | Configuration for remote execution capabilities. If not set the capabilities service will inform the client that remote execution is not supported. | +| `remote_cache_compression` | boolean | — | — | Whether this instance supports Bazel remote cache compression. When enabled, the capabilities service advertises zstd wire compression and the ByteStream/CAS services accept REAPI compressed-blobs/zstd data. | + +## ExecutionServiceConfig + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `instance_name` | string | — | `""` | Used when the config references `instance_name` in the protocol. | +| `cas_store` | string | Yes | — | The store name referenced in the `stores` map in the main config. This store name referenced here may be reused multiple times. This value must be a CAS store reference. | +| `scheduler` | string | Yes | — | The scheduler name referenced in the `schedulers` map in the main config. | + +## ByteStreamServiceConfig + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `instance_name` | string | — | `""` | Used when the config references `instance_name` in the protocol. | +| `cas_store` | string | Yes | — | Name of the store in the "stores" configuration. | +| `max_bytes_per_stream` | integer (uint) | — | 64KiB | Max number of bytes to send on each grpc stream chunk. According to [https://github.com/grpc/grpc.github.io/issues/371](https://github.com/grpc/grpc.github.io/issues/371) 16KiB - 64KiB is optimal. | +| `persist_stream_on_disconnect_timeout_s` | integer (uint) | — | 10 seconds | In the event a client disconnects while uploading a blob, we will hold the internal stream open for this many seconds before closing it. This allows clients that disconnect to reconnect and continue uploading the same blob. | + +## FetchServiceConfig + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `instance_name` | string | — | `""` | Used when the config references `instance_name` in the protocol. | +| `fetch_store` | string | Yes | — | The store name referenced in the `stores` map in the main config. This store name referenced here may be reused multiple times. | + +## PushServiceConfig + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `instance_name` | string | — | `""` | Used when the config references `instance_name` in the protocol. | +| `push_store` | string | Yes | — | The store name referenced in the `stores` map in the main config. This store name referenced here may be reused multiple times. | +| `read_only` | boolean | — | `false` | Whether the Action Cache store may be written to, this if set to false it is only possible to read from the Action Cache. | + +## WorkerApiConfig + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `scheduler` | string | Yes | — | The scheduler name referenced in the `schedulers` map in the main config. | + +## BepConfig + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `store` | string | Yes | — | The store to publish build events to. The store name referenced in the `stores` map in the main config. | + +## AdminConfig + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `path` | string | — | "/admin" | Path to register the admin API. If path is "/admin", and your domain is "example.com", you can reach the endpoint with: [http://example.com/admin](http://example.com/admin). | + +## HealthConfig + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `path` | string | — | "/status" | Path to register the health status check. If path is "/status", and your domain is "example.com", you can reach the endpoint with: [http://example.com/status](http://example.com/status). | +| `timeout_seconds` | integer (uint64) | — | 5s | Timeout on health checks. | + +## ErrorCode + +The possible error codes that might occur on an upstream request. + +| Value | Description | +| --- | --- | +| `"Cancelled"` | | +| `"Unknown"` | | +| `"InvalidArgument"` | | +| `"DeadlineExceeded"` | | +| `"NotFound"` | | +| `"AlreadyExists"` | | +| `"PermissionDenied"` | | +| `"ResourceExhausted"` | | +| `"FailedPrecondition"` | | +| `"Aborted"` | | +| `"OutOfRange"` | | +| `"Unimplemented"` | | +| `"Internal"` | | +| `"Unavailable"` | | +| `"DataLoss"` | | +| `"Unauthenticated"` | | + +## Lz4Config + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `block_size` | integer (uint32) | — | 65536 (64k) | Size of the blocks to compress. Higher values require more ram, but might yield slightly better compression ratios. | +| `max_decode_block_size` | integer (uint32) | — | value in `block_size` | Maximum size allowed to attempt to deserialize data into. This is needed because the `block_size` is embedded into the data so if there was a bad actor, they could upload an extremely large `block_size`'ed entry and we'd allocate a large amount of memory when retrieving the data. To prevent this from happening, we allow you to specify the maximum that we'll attempt to deserialize. | + +## ClientTlsConfig + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `ca_file` | string | — | — | Path to the certificate authority to use to validate the remote. | +| `cert_file` | string | — | — | Path to the certificate file for client authentication. | +| `key_file` | string | — | — | Path to the private key file for client authentication. | +| `use_native_roots` | boolean | — | false | If set the client will use the native roots for TLS connections. | + +## UploadCacheResultsStrategy + +| Value | Description | +| --- | --- | +| `"success_only"` | Only upload action results with an exit code of 0. | +| `"never"` | Don't upload any action results. | +| `"everything"` | Upload all action results that complete. | +| `"failures_only"` | Only upload action results that fail. | + +## ExperimentalRedisSchedulerBackend + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `redis_store` | string | Yes | — | A reference to the Redis store to use for the scheduler. Note: This MUST resolve to a `RedisSpec`. | + +## PlatformPropertyAddition + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `name` | string | Yes | — | The name of the property to add. | +| `value` | string | Yes | — | The value to assign to the property. | + +## PlatformPropertyReplacement + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `name` | string | Yes | — | The name of the property to replace. | +| `value` | string | — | — | The value to match against, if unset then any instance matches. | +| `new_name` | string | Yes | — | The new name of the property. | +| `new_value` | string | — | — | The value to assign to the property, if unset will remain the same. | + +## HttpCompressionConfig + +Note: Compressing data in the cloud rarely has a benefit, since most +cloud providers have very high bandwidth backplanes. However, for +clients not inside the data center, it might be a good idea to +compress data to and from the cloud. This will however come at a high +CPU and performance cost. If you are making remote execution share the +same CAS/AC servers as client's remote cache, you can create multiple +services with different compression settings that are served on +different ports. Then configure the non-cloud clients to use one port +and cloud-clients to use another. + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `send_compression_algorithm` | [HttpCompressionAlgorithm](#httpcompressionalgorithm) | — | `HttpCompressionAlgorithm::None` | The compression algorithm that the server will use when sending responses to clients. Enabling this will likely save a lot of data transfer, but will consume a lot of CPU and add a lot of latency. see: [https://github.com/tracemachina/nativelink/issues/109](https://github.com/tracemachina/nativelink/issues/109) | +| `accepted_compression_algorithms` | array of [HttpCompressionAlgorithm](#httpcompressionalgorithm) | Yes | {no supported compression} | The compression algorithm that the server will accept from clients. The server will broadcast the supported compression algorithms to clients and the client will choose which compression algorithm to use. Enabling this will likely save a lot of data transfer, but will consume a lot of CPU and add a lot of latency. see: [https://github.com/tracemachina/nativelink/issues/109](https://github.com/tracemachina/nativelink/issues/109) | + +## HttpServerConfig + +Advanced HTTP configuration. These generally should not be set. +For documentation on these settings, see the hyper documentation: +See: [hyper HTTP server docs](https://docs.rs/hyper/latest/hyper/server/conn/struct.Http.html) + +Note: All of these default to the default values from hyper unless otherwise +specified. + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `http2_keep_alive_interval` | integer (uint32) | — | — | Interval to send keep-alive pings via HTTP2. Note: This is in seconds. | +| `experimental_http2_max_pending_accept_reset_streams` | integer (uint32) | — | — | | +| `experimental_http2_initial_stream_window_size` | integer (uint32) | — | — | | +| `experimental_http2_initial_connection_window_size` | integer (uint32) | — | — | | +| `experimental_http2_adaptive_window` | boolean | — | — | | +| `experimental_http2_max_frame_size` | integer (uint32) | — | — | | +| `experimental_http2_max_concurrent_streams` | integer (uint32) | — | — | | +| `experimental_http2_keep_alive_timeout_s` | integer (uint32) | — | — | | +| `experimental_http2_max_send_buf_size` | integer (uint32) | — | — | | +| `experimental_http2_enable_connect_protocol` | boolean | — | — | | +| `experimental_http2_max_header_list_size` | integer (uint32) | — | — | | + +## TlsConfig + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `cert_file` | string | Yes | — | Path to the certificate file. | +| `key_file` | string | Yes | — | Path to the private key file. | +| `client_ca_file` | string | — | — | Path to the certificate authority for mTLS, if client authentication is required for this endpoint. | +| `client_crl_file` | string | — | — | Path to the certificate revocation list for mTLS, if client authentication is required for this endpoint. | + +## CasChunkingConfig + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `index_store` | string | — | — | The store name referenced in the `stores` map in the main config used to persist blob-to-chunks layouts. Keys are the digests of the original blobs and values are serialized chunk layouts (which do not hash to those digests), so this store MUST NOT perform content digest verification and MUST NOT be the same store as `cas_store` — writing layouts into the CAS would overwrite blob content. Using the same store name as `cas_store` is rejected at startup. | +| `avg_chunk_size_bytes` | integer (uint64) | — | 524288 (512 KiB) | The average chunk size in bytes advertised to clients through the `FastCDC` 2020 capability parameters and used for server-side chunking in `SplitBlob`. Clients derive the minimum and maximum chunk sizes from this value (avg / 4 and avg * 4). The value must be between 1 KiB and 1 MiB. | +| `max_chunk_count` | integer (uint64) | — | 50000 | Maximum number of chunks accepted in a `SpliceBlob` request or produced by on-demand chunking in `SplitBlob`. Blobs that would produce more chunks are served without chunking (`SplitBlob` returns `NOT_FOUND` and clients fall back to a regular download). This bounds the size of stored chunk layouts and of `SplitBlobResponse` messages (roughly 80-140 bytes per chunk). At the default average chunk size the default cap supports blobs up to ~25 GiB; note that values above ~50000 may produce responses that exceed default gRPC message size limits on clients. | + +## CapabilitiesRemoteExecutionConfig + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `scheduler` | string | Yes | — | Scheduler used to configure the capabilities of remote execution. | + +## HttpCompressionAlgorithm + +| Value | Description | +| --- | --- | +| `"none"` | No compression. | +| `"gzip"` | Zlib compression. | + +## Reading the source + +If anything here disagrees with the binary, the source wins: + +- [`stores.rs`](https://github.com/TraceMachina/nativelink/tree/v1.6.1/nativelink-config/src/stores.rs) +- [`cas_server.rs`](https://github.com/TraceMachina/nativelink/tree/v1.6.1/nativelink-config/src/cas_server.rs) +- [`schedulers.rs`](https://github.com/TraceMachina/nativelink/tree/v1.6.1/nativelink-config/src/schedulers.rs) diff --git a/web/apps/docs/content/docs/reference/nativelink-config/v1.6.2.mdx b/web/apps/docs/content/docs/reference/nativelink-config/v1.6.2.mdx new file mode 100644 index 000000000..beb6bb0a7 --- /dev/null +++ b/web/apps/docs/content/docs/reference/nativelink-config/v1.6.2.mdx @@ -0,0 +1,1537 @@ +--- +title: Configuration reference +description: Every knob in the NativeLink JSON5 configuration — types, defaults, and links to source, autogenerated from the Rust config crate. +full: true +--- + +{/* AUTOGENERATED — do not edit by hand. + Source: nativelink-config @ v1.6.2 (9717f2fb) + Regenerate from web/: bun --filter @nativelink/docs gen:config-reference */} + + + +This is the canonical NativeLink configuration reference for **v1.6.2**. +It is autogenerated from the Rust config crate +([`nativelink-config/src`](https://github.com/TraceMachina/nativelink/tree/v1.6.2/nativelink-config/src)) via the `build-schema` binary, so +it can never drift from what the binary actually deserializes. + +## Top-level fields + +The root object (`CasConfig`) accepts the following fields: + +| Field | Type | Required | Description | +| --- | --- | --- | --- | +| `stores` | array of [NamedStoreConfig](#namedstoreconfig) | Yes | List of stores available to use in this config. The keys can be used in other configs when needing to reference a store. | +| `workers` | array of [WorkerConfig](#workerconfig) | — | Worker configurations used to execute jobs. | +| `schedulers` | array of [NamedSchedulerConfig](#namedschedulerconfig) | — | List of schedulers available to use in this config. The keys can be used in other configs when needing to reference a scheduler. | +| `servers` | array of [ServerConfig](#serverconfig) | Yes | Servers to setup for this process. | +| `experimental_origin_events` | [OriginEventsSpec](#origineventsspec) | — | Experimental - Origin events configuration. This is the service that will collect and publish nativelink events to a store for processing by an external service. | +| `global` | [GlobalConfig](#globalconfig) | — | Any global configurations that apply to all modules live here. | + +## Configuration types + +Every type reachable from the root configuration, in reading order. + +## NamedStoreConfig + +**Common fields** + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `name` | string | Yes | — | | + +Plus exactly one of the following variants (the key selects the variant): + +### `cache_metrics` + +Cache metrics store wraps another store and emits low-cardinality +OpenTelemetry cache operation metrics for the wrapped store. + +This wrapper is opt-in. Stores that are not explicitly wrapped by +`cache_metrics` are constructed exactly as they are without this +wrapper and do not pay its hot-path timing or recording cost. + +**Example JSON5 config:** +```json5 +"cache_metrics": { + "cache_type": "cas", + "backend": { + "filesystem": { + "content_path": "~/.cache/nativelink/content_path-cas", + "temp_path": "~/.cache/nativelink/tmp_path-cas" + } + } +} +``` + +**Type:** [CacheMetricsSpec](#cachemetricsspec) + +### `memory` + +Memory store will store all data in a hash map in memory. + +**Example JSON5 config:** +```json5 +"memory": { + "eviction_policy": { + "max_bytes": "10mb", + } +} +``` + +**Type:** [MemorySpec](#memoryspec) + +### `experimental_cloud_object_store` + +A generic blob store that will store files on the cloud +provider. This configuration will never delete files, so you are +responsible for purging old files in other ways. +It supports the following backends: + +1. **Amazon S3:** + S3 store will use Amazon's S3 service as a backend to store + the files. This configuration can be used to share files + across multiple instances. Uses system certificates for TLS + verification via `rustls-platform-verifier`. + + **Example JSON5 config:** + ```json5 + "experimental_cloud_object_store": { + "provider": "aws", + "region": "eu-north-1", + "bucket": "crossplane-bucket-af79aeca9", + "key_prefix": "test-prefix-index/", + "retry": { + "max_retries": 6, + "delay": 0.3, + "jitter": 0.5 + }, + "multipart_max_concurrent_uploads": 10 + } + ``` + +2. **Google Cloud Storage:** + GCS store uses Google's GCS service as a backend to store + the files. This configuration can be used to share files + across multiple instances. + + **Example JSON5 config:** + ```json5 + "experimental_cloud_object_store": { + "provider": "gcs", + "bucket": "test-bucket", + "key_prefix": "test-prefix-index/", + "retry": { + "max_retries": 6, + "delay": 0.3, + "jitter": 0.5 + }, + "multipart_max_concurrent_uploads": 10 + } + ``` + +3. **Azure Blob Store:** + Azure Blob store will use Microsoft's Azure Blob service as a + backend to store the files. This configuration can be used to + share files across multiple instances. + + **Example JSON5 config:** + ```json5 + "experimental_cloud_object_store": { + "provider": "azure", + "account_name": "cloudshell1393657559", + "container": "simple-test-container", + "key_prefix": "folder/", + "retry": { + "max_retries": 6, + "delay": 0.3, + "jitter": 0.5 + }, + "multipart_max_concurrent_uploads": 10 + } + ``` + +4. **NetApp ONTAP S3:** + NetApp ONTAP S3 store will use ONTAP's S3-compatible storage as a backend + to store files. This store is specifically configured for ONTAP's S3 requirements + including custom TLS configuration, credentials management, and proper vserver + configuration. + + This store uses AWS environment variables for credentials: + - `AWS_ACCESS_KEY_ID` + - `AWS_SECRET_ACCESS_KEY` + - `AWS_DEFAULT_REGION` + + **Example JSON5 config:** + ```json5 + "experimental_cloud_object_store": { + "provider": "ontap", + "endpoint": "https://ontap-s3-endpoint:443", + "vserver_name": "your-vserver", + "bucket": "your-bucket", + "root_certificates": "/path/to/certs.pem", // Optional + "key_prefix": "test-prefix/", // Optional + "retry": { + "max_retries": 6, + "delay": 0.3, + "jitter": 0.5 + }, + "multipart_max_concurrent_uploads": 10 + } + ``` + +5. **Cloudflare R2:** + R2 store uses Cloudflare's R2 service as a backend. R2 speaks the + S3 API, so this is a thin wrapper that derives the account-scoped + endpoint (`https://{account_id}.r2.cloudflarestorage.com`) for you. + + **Example JSON5 config:** + ```json5 + "experimental_cloud_object_store": { + "provider": "r2", + "account_id": "a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4", + "bucket": "nativelink-cas", + "key_prefix": "test-prefix/", + "retry": { + "max_retries": 6, + "delay": 0.3, + "jitter": 0.5 + }, + "multipart_max_concurrent_uploads": 10 + } + ``` + +6. **Oracle Cloud Infrastructure (OCI) Object Storage:** + OCI store uses Oracle Cloud Infrastructure's S3-compatible Object + Storage API. The path-style endpoint is derived from your Object + Storage `namespace` and `region` as + `https://{namespace}.compat.objectstorage.{region}.oci.customer-oci.com`. + Authenticate with a Customer Secret Key (Access Key/Secret Key pair + created under User Settings -> Customer secret keys in the OCI + console); the secret cannot be retrieved after generation, so read + it from an env var via shellexpand. + + **Example JSON5 config:** + ```json5 + "experimental_cloud_object_store": { + "provider": "oci", + "namespace": "your-object-storage-namespace", + "region": "us-phoenix-1", + "bucket": "nativelink-cas", + "access_key_id": "oci_access_key_id", + "secret_access_key": "oci_secret_access_key", + "key_prefix": "test-prefix/", + "retry": { + "max_retries": 6, + "delay": 0.3, + "jitter": 0.5 + } + } + ``` + +**Type:** [ExperimentalCloudObjectSpec](#experimentalcloudobjectspec) + +### `ontap_s3_existence_cache` + +ONTAP S3 Existence Cache provides a caching layer on top of the ONTAP S3 store +to optimize repeated existence checks. It maintains an in-memory cache of object +digests and periodically syncs this cache to disk for persistence. + +The cache helps reduce latency for repeated calls to check object existence, +while still ensuring eventual consistency with the underlying ONTAP S3 store. + +Example JSON5 config: +```json5 +"ontap_s3_existence_cache": { + "index_path": "/path/to/cache/index.json", + "sync_interval_seconds": 300, + "backend": { + "endpoint": "https://ontap-s3-endpoint:443", + "vserver_name": "your-vserver", + "bucket": "your-bucket", + "key_prefix": "test-prefix/" + } +} +``` + +**Type:** [OntapS3ExistenceCacheSpec](#ontaps3existencecachespec) + +### `verify` + +Verify store is used to apply verifications to an underlying +store implementation. It is strongly encouraged to validate +as much data as you can before accepting data from a client, +failing to do so may cause the data in the store to be +populated with invalid data causing all kinds of problems. + +The suggested configuration is to have the CAS validate the +hash and size and the AC validate nothing. + +**Example JSON5 config:** +```json5 +"verify": { + "backend": { + "memory": { + "eviction_policy": { + "max_bytes": "500mb" + } + }, + }, + "verify_size": true, + "verify_hash": true +} +``` + +**Type:** [VerifySpec](#verifyspec) + +### `completeness_checking` + +Completeness checking store verifies if the +output files & folders exist in the CAS before forwarding +the request to the underlying store. +Note: This store should only be used on AC stores. + +**Example JSON5 config:** +```json5 +"completeness_checking": { + "backend": { + "filesystem": { + "content_path": "~/.cache/nativelink/content_path-ac", + "temp_path": "~/.cache/nativelink/tmp_path-ac", + "eviction_policy": { + "max_bytes": "500mb", + } + } + }, + "cas_store": { + "ref_store": { + "name": "CAS_MAIN_STORE" + } + } +} +``` + +**Type:** [CompletenessCheckingSpec](#completenesscheckingspec) + +### `compression` + +A compression store that will compress the data inbound and +outbound. There will be a non-trivial cost to compress and +decompress the data, but in many cases if the final store is +a store that requires network transport and/or storage space +is a concern it is often faster and more efficient to use this +store before those stores. + +**Example JSON5 config:** +```json5 +"compression": { + "compression_algorithm": { + "lz4": {} + }, + "backend": { + "filesystem": { + "content_path": "/tmp/nativelink/data/content_path-cas", + "temp_path": "/tmp/nativelink/data/tmp_path-cas", + "eviction_policy": { + "max_bytes": "2gb", + } + } + } +} +``` + +**Type:** [CompressionSpec](#compressionspec) + +### `dedup` + +A dedup store will take the inputs and run a rolling hash +algorithm on them to slice the input into smaller parts then +run a sha256 algorithm on the slice and if the object doesn't +already exist, upload the slice to the `content_store` using +a new digest of just the slice. Once all parts exist, an +Action-Cache-like digest will be built and uploaded to the +`index_store` which will contain a reference to each +chunk/digest of the uploaded file. Downloading a request will +first grab the index from the `index_store`, and forward the +download content of each chunk as if it were one file. + +This store is exceptionally good when the following conditions +are met: +* Content is mostly the same (inserts, updates, deletes are ok) +* Content is not compressed or encrypted +* Uploading or downloading from `content_store` is the bottleneck. + +Note: This store pairs well when used with `CompressionSpec` as +the `content_store`, but never put `DedupSpec` as the backend of +`CompressionSpec` as it will negate all the gains. + +Note: When running `.has()` on this store, it will only check +to see if the entry exists in the `index_store` and not check +if the individual chunks exist in the `content_store`. + +**Example JSON5 config:** +```json5 +"dedup": { + "index_store": { + "memory": { + "eviction_policy": { + "max_bytes": "1GB", + } + } + }, + "content_store": { + "compression": { + "compression_algorithm": { + "lz4": {} + }, + "backend": { + "fast_slow": { + "fast": { + "memory": { + "eviction_policy": { + "max_bytes": "500MB", + } + } + }, + "slow": { + "filesystem": { + "content_path": "/tmp/nativelink/data/content_path-content", + "temp_path": "/tmp/nativelink/data/tmp_path-content", + "eviction_policy": { + "max_bytes": "2gb" + } + } + } + } + } + } + } +} +``` + +**Type:** [DedupSpec](#dedupspec) + +### `existence_cache` + +Existence store will wrap around another store and cache calls +to has so that subsequent `has_with_results` calls will be +faster. This is useful for cases when you have a store that +is slow to respond to has calls. +Note: This store should only be used on CAS stores. + +**Example JSON5 config:** +```json5 +"existence_cache": { + "backend": { + "memory": { + "eviction_policy": { + "max_bytes": "500mb", + } + } + }, + // Note this is the existence store policy, not the backend policy + "eviction_policy": { + "max_seconds": 100, + } +} +``` + +**Type:** [ExistenceCacheSpec](#existencecachespec) + +### `fast_slow` + +`FastSlow` store will first try to fetch the data from the `fast` +store and then if it does not exist try the `slow` store. +When the object does exist in the `slow` store, it will copy +the data to the `fast` store while returning the data. +This store should be thought of as a store that "buffers" +the data to the `fast` store. +On uploads it will mirror data to both `fast` and `slow` stores. + +WARNING: If you need data to always exist in the `slow` store +for something like remote execution, be careful because this +store will never check to see if the objects exist in the +`slow` store if it exists in the `fast` store (i.e. it assumes +that if an object exists in the `fast` store it will exist in +the `slow` store). + +***Example JSON5 config:*** +```json5 +"fast_slow": { + "fast": { + "filesystem": { + "content_path": "/tmp/nativelink/data/content_path-index", + "temp_path": "/tmp/nativelink/data/tmp_path-index", + "eviction_policy": { + "max_bytes": "500mb", + } + } + }, + "slow": { + "filesystem": { + "content_path": "/tmp/nativelink/data/content_path-index", + "temp_path": "/tmp/nativelink/data/tmp_path-index", + "eviction_policy": { + "max_bytes": "500mb", + } + } + } +} +``` + +**Type:** [FastSlowSpec](#fastslowspec) + +### `shard` + +Shards the data to multiple stores. This is useful for cases +when you want to distribute the load across multiple stores. +The digest hash is used to determine which store to send the +data to. + +**Example JSON5 config:** +```json5 +"shard": { + "stores": [ + { + "store": { + "memory": { + "eviction_policy": { + "max_bytes": "10mb" + }, + }, + }, + "weight": 1 + }] +} +``` + +**Type:** [ShardSpec](#shardspec) + +### `filesystem` + +Stores the data on the filesystem. This store is designed for +local persistent storage. Restarts of this program should restore +the previous state, meaning anything uploaded will be persistent +as long as the filesystem integrity holds. + +**Example JSON5 config:** +```json5 +"filesystem": { + "content_path": "/tmp/nativelink/data-worker-test/content_path-cas", + "temp_path": "/tmp/nativelink/data-worker-test/tmp_path-cas", + "eviction_policy": { + "max_bytes": "10gb", + } +} +``` + +**Type:** [FilesystemSpec](#filesystemspec) + +### `ref_store` + +Store used to reference a store in the root store manager. +This is useful for cases when you want to share a store in different +nested stores. Example, you may want to share the same memory store +used for the action cache, but use a `FastSlowSpec` and have the fast +store also share the memory store for efficiency. + +**Example JSON5 config:** +```json5 +"ref_store": { + "name": "FS_CONTENT_STORE" +} +``` + +**Type:** [RefSpec](#refspec) + +### `size_partitioning` + +Uses the size field of the digest to separate which store to send the +data. This is useful for cases when you'd like to put small objects +in one store and large objects in another store. This should only be +used if the size field is the real size of the content, in other +words, don't use on AC (Action Cache) stores. Any store where you can +safely use `VerifySpec.verify_size = true`, this store should be safe +to use (i.e. CAS stores). + +**Example JSON5 config:** +```json5 +"size_partitioning": { + "size": "128mib", + "lower_store": { + "memory": { + "eviction_policy": { + "max_bytes": "${NATIVELINK_CAS_MEMORY_CONTENT_LIMIT:-100mb}" + } + } + }, + "upper_store": { + /// This store discards data larger than 128mib. + "noop": {} + } +} +``` + +**Type:** [SizePartitioningSpec](#sizepartitioningspec) + +### `grpc` + +This store will pass-through calls to another gRPC store. This store +is not designed to be used as a sub-store of another store, but it +does satisfy the interface and will likely work. + +One major GOTCHA is that some stores use a special function on this +store to get the size of the underlying object, which is only reliable +when this store is serving the a CAS store, not an AC store. If using +this store directly without being a child of any store there are no +side effects and is the most efficient way to use it. + +**Example JSON5 config:** +```json5 +"grpc": { + "instance_name": "main", + "endpoints": [ + {"address": "grpc://${CAS_ENDPOINT:-127.0.0.1}:50051"} + ], + "connections_per_endpoint": "5", + "rpc_timeout_s": "5m", + "store_type": "ac", + // Static headers attached to every outgoing request to the upstream + // remote cache. Useful for fixed service-account credentials. + "headers": { + "authorization": "Bearer my-static-token" + }, + // Header names to copy from the inbound client request and forward to + // the upstream remote cache. Use this to pass through dynamic + // credentials such as a JWT sent by the build client. + "forward_headers": ["authorization", "x-custom-token"] +} +``` + +**Type:** [GrpcSpec](#grpcspec) + +### `redis_store` + +Stores data in any stores compatible with Redis APIs. + +Pairs well with `SizePartitioning` and/or `FastSlow` stores. +Ideal for accepting small object sizes as most Redis store +services have a max file upload of between 256Mb-512Mb. + +**Example JSON5 config:** +```json5 +"redis_store": { + "addresses": [ + "redis://127.0.0.1:6379/", + ], + "max_client_permits": 1000, +} +``` + +**Type:** [RedisSpec](#redisspec) + +### `noop` + +Noop store is a store that sends streams into the void and all data +retrieval will return 404 (`NotFound`). This can be useful for cases +where you may need to partition your data and part of your data needs +to be discarded. + +**Example JSON5 config:** +```json5 +"noop": {} +``` + +**Type:** [NoopSpec](#noopspec) + +### `experimental_mongo` + +Experimental `MongoDB` store implementation. + +This store uses `MongoDB` as a backend for storing data. It supports +both CAS (Content Addressable Storage) and scheduler data with +optional change streams for real-time updates. + +**Example JSON5 config:** +```json5 +"experimental_mongo": { + "connection_string": "mongodb://localhost:27017", + "database": "nativelink", + "cas_collection": "cas", + "key_prefix": "cas:", + "read_chunk_size": 65536, + "max_concurrent_uploads": 10, + "enable_change_streams": false, + "max_requests": "100" +} +``` + +**Type:** [ExperimentalMongoSpec](#experimentalmongospec) + +## WorkerConfig + +Plus exactly one of the following variants (the key selects the variant): + +### `local` + +A worker type that executes jobs locally on this machine. + +**Type:** [LocalWorkerConfig](#localworkerconfig) + +## NamedSchedulerConfig + +**Common fields** + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `name` | string | Yes | — | | + +Plus exactly one of the following variants (the key selects the variant): + +### `simple` + +**Type:** [SimpleSpec](#simplespec) + +### `grpc` + +**Type:** [SchedulerGrpcSpec](#schedulergrpcspec) + +### `cache_lookup` + +**Type:** [CacheLookupSpec](#cachelookupspec) + +### `property_modifier` + +**Type:** [PropertyModifierSpec](#propertymodifierspec) + +### `historical_resource` + +**Type:** [HistoricalResourceSpec](#historicalresourcespec) + +## ServerConfig + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `name` | string | — | {index of server in config} | Name of the server. This is used to help identify the service for telemetry and logs. | +| `listener` | [ListenerConfig](#listenerconfig) | Yes | — | Configuration | +| `services` | [ServicesConfig](#servicesconfig) | — | — | Services to attach to server. | +| `experimental_identity_header` | [IdentityHeaderSpec](#identityheaderspec) | — | {see `IdentityHeaderSpec`} | The config related to identifying the client. | + +## OriginEventsSpec + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `publisher` | [OriginEventsPublisherSpec](#origineventspublisherspec) | Yes | — | The publisher configuration for origin events. | +| `max_event_queue_size` | integer (uint) | — | 65536 (zero defaults to this) | The maximum number of events to queue before applying back pressure. IMPORTANT: Backpressure causes all clients to slow down significantly. Zero is default. | + +## GlobalConfig + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `max_open_files` | integer (uint) | Yes | 24576 (= 24 * 1024) | Maximum number of open files that can be opened at one time. This value is not strictly enforced, it is a best effort. Some internal libraries open files or read metadata from a files which do not obey this limit, however the vast majority of cases will have this limit be honored. This value must be larger than `ulimit -n` to have any effect. Any network open file descriptors is not counted in this limit, but is counted in the kernel limit. It is a good idea to set a very large `ulimit -n`. Note: This value must be greater than 10. | +| `default_digest_hash_function` | [ConfigDigestHashFunction](#configdigesthashfunction) | — | `ConfigDigestHashFunction::sha256` | Default hash function to use while uploading blobs to the CAS when not set by client. | +| `default_digest_size_health_check` | integer (uint) | — | 1024*1024 (1MiB) | Default digest size to use for health check when running diagnostics checks. Health checks are expected to use this size for filling a buffer that is used for creation of digest. | + +## CacheMetricsSpec + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `cache_type` | string | Yes | — | Low-cardinality cache type label for metrics, for example `cas` or `ac`. | +| `backend` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | Store to wrap with cache operation metrics. | + +## MemorySpec + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `eviction_policy` | [EvictionPolicy](#evictionpolicy) | — | — | Policy used to evict items out of the store. Failure to set this value will cause items to never be removed from the store causing infinite memory usage. | + +## ExperimentalCloudObjectSpec + +See [`experimental_cloud_object_store`](#experimental_cloud_object_store-1) for details + +## OntapS3ExistenceCacheSpec + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `index_path` | string | Yes | — | | +| `sync_interval_seconds` | integer (uint32) | Yes | — | | +| `backend` | [ExperimentalOntapS3Spec](#experimentalontaps3spec) | Yes | — | | + +## VerifySpec + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `backend` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | The underlying store wrap around. All content will first flow through self before forwarding to backend. In the event there is an error detected in self, the connection to the backend will be terminated, and early termination should always cause updates to fail on the backend. | +| `verify_size` | boolean | — | `false` | If set the store will verify the size of the data before accepting an upload of data. | +| `verify_hash` | boolean | — | `false` | If the data should be hashed and verify that the key matches the computed hash. The hash function is automatically determined based request and if not set will use the global default. | + +## CompletenessCheckingSpec + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `backend` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | The underlying store that will have it's results validated before sending to client. | +| `cas_store` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | When a request is made, the results are decoded and all output digests/files are verified to exist in this CAS store before returning success. | + +## CompressionSpec + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `backend` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | The underlying store wrap around. All content will first flow through self before forwarding to backend. In the event there is an error detected in self, the connection to the backend will be terminated, and early termination should always cause updates to fail on the backend. | +| `compression_algorithm` | [CompressionAlgorithm](#compressionalgorithm) | Yes | — | The compression algorithm to use. | + +## DedupSpec + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `index_store` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | Store used to store the index of each dedup slice. This store should generally be fast and small. | +| `content_store` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | The store where the individual chunks will be uploaded. This store should generally be the slower & larger store. | +| `min_size` | integer (uint32) | — | 64k | Minimum size that a chunk will be when slicing up the content. Note: This setting can be increased to improve performance because it will actually not check this number of bytes when deciding where to partition the data. | +| `normal_size` | integer (uint32) | — | 256k | A best-effort attempt will be made to keep the average size of the chunks to this number. It is not a guarantee, but a slight attempt will be made. | +| `max_size` | integer (uint32) | — | 512k | Maximum size a chunk is allowed to be. | +| `max_concurrent_fetch_per_get` | integer (uint32) | — | 10 | Due to implementation detail, we want to prefer to download the first chunks of the file so we can stream the content out and free up some of our buffers. This configuration will be used to restrict the number of concurrent chunk downloads at a time per `get()` request. | + +## ExistenceCacheSpec + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `backend` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | The underlying store wrap around. All content will first flow through self before forwarding to backend. In the event there is an error detected in self, the connection to the backend will be terminated, and early termination should always cause updates to fail on the backend. | +| `eviction_policy` | [EvictionPolicy](#evictionpolicy) | — | — | Policy used to evict items out of the store. Failure to set this value will cause items to never be removed from the store causing infinite memory usage. | + +## FastSlowSpec + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `fast` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | Fast store that will be attempted to be contacted before reaching out to the `slow` store. | +| `fast_direction` | [StoreDirection](#storedirection) | — | `"both"` | How to handle the fast store. This can be useful to set to Get for worker nodes such that results are persisted to the slow store only. | +| `slow` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | If the object does not exist in the `fast` store it will try to get it from this store. | +| `slow_direction` | [StoreDirection](#storedirection) | — | `"both"` | How to handle the slow store. This can be useful if creating a diode and you wish to have an upstream read only store. | +| `bypass_dedup_threshold_bytes` | integer (uint64) | — | disabled (0) | Reads of blobs at or above this size skip the leader/follower dedup map and stream straight from the slow store without populating the fast tier. `0` (the default) disables the bypass: every read goes through dedup, matching the prior behaviour. Enable it by setting a threshold — 256 MiB is a reasonable starting point for backends where large-blob dedup is a net loss (followers tend to time out anyway), but the right value is workload-dependent. | + +## ShardSpec + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `stores` | array of [ShardConfig](#shardconfig) | Yes | — | Stores to shard the data to. | + +## FilesystemSpec + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `content_path` | string | Yes | — | Path on the system where to store the actual content. This is where the bulk of the data will be placed. On service boot this folder will be scanned and all files will be added to the cache. In the event one of the files doesn't match the criteria, the file will be deleted. | +| `temp_path` | string | Yes | — | A temporary location of where files that are being uploaded or deleted will be placed while the content cannot be guaranteed to be accurate. This location must be on the same block device as `content_path` so atomic moves can happen (i.e. move without copy). All files in this folder will be deleted on every startup. | +| `read_buffer_size` | integer (uint32) | — | 32k | Buffer size to use when reading files. Generally this should be left to the default value except for testing. | +| `eviction_policy` | [EvictionPolicy](#evictionpolicy) | — | — | Policy used to evict items out of the store. Failure to set this value will cause items to never be removed from the store causing infinite memory usage. | +| `block_size` | integer (uint64) | — | 4kb | The block size of the filesystem for the running machine value is used to determine an entry's actual size on disk consumed For a 4KB block size filesystem, a 1B file actually consumes 4KB | +| `max_concurrent_writes` | integer (uint) | — | unlimited | Maximum number of concurrent write operations allowed. Each write involves streaming data to a temp file and calling `sync_all()`, which can saturate disk I/O when many writes happen simultaneously. Limiting concurrency prevents disk saturation from blocking the async runtime. A value of 0 means unlimited (no concurrency limit). | + +## RefSpec + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `name` | string | Yes | — | Name of the store under the root "stores" config object. | + +## SizePartitioningSpec + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `size` | integer (uint64) | Yes | — | Size to partition the data on. | +| `lower_store` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | Store to send data when object is < (less than) size. | +| `upper_store` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | Store to send data when object is >= (less than eq) size. | + +## GrpcSpec + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `instance_name` | string | — | `""` | Instance name for gRPC calls. Proxy calls will have the `instance_name` changed to this. | +| `endpoints` | array of [GrpcEndpoint](#grpcendpoint) | Yes | — | The endpoint of the grpc connection. | +| `store_type` | [StoreType](#storetype) | Yes | — | The type of the upstream store, this ensures that the correct server calls are made. | +| `retry` | [Retry](#retry) | — | — | Retry configuration to use when a network request fails. | +| `max_concurrent_requests` | integer (uint) | — | `0` | Limit the number of simultaneous upstream requests to this many. A value of zero is treated as unlimited. If the limit is reached the request is queued. | +| `connections_per_endpoint` | integer (uint) | — | 1 | The number of connections to make to each specified endpoint to balance the load over multiple TCP connections. | +| `rpc_timeout_s` | integer (uint64) | — | 0 (disabled) | Maximum time (seconds) allowed for a single RPC request (e.g. a `ByteStream.Write` call) before it is cancelled. | +| `use_legacy_resource_names` | boolean | — | false | Use legacy `ByteStream` resource name format, omitting the digest function component from the path. | +| `headers` | map of string to string | — | — | Static headers to attach to every outgoing gRPC request sent to this store's upstream endpoints. Useful for fixed authentication tokens (e.g. `{"authorization": "Bearer "}`) and other static metadata. | +| `forward_headers` | array of string | — | — | Header names to forward from the incoming client request to every outgoing upstream request. The header value is taken from the client request that triggered this store operation. Use this to pass through dynamic credentials such as JWT tokens sent by build clients. | +| `experimental_read_batching` | [GrpcReadBatchingConfig](#grpcreadbatchingconfig) | — | unset (disabled) | Optional and experimental: coalesce small-blob reads into `BatchReadBlobs` RPCs instead of issuing one `ByteStream` `Read` stream per blob. Each `ByteStream` read carries a fixed per-RPC cost, so batching many small reads into a single `BatchReadBlobs` request can dramatically reduce read latency for small blobs. | + +## RedisSpec + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `addresses` | array of string | Yes | — | The hostname or IP address of the Redis server. Ex: `["redis://username:password@redis-server-url:6380/99"]` 99 Represents database ID, 6380 represents the port. | +| `response_timeout_s` | integer (uint64) | — | 10 | DEPRECATED: use `command_timeout_ms` The response timeout for the Redis connection in seconds. | +| `connection_timeout_s` | integer (uint64) | — | 10 | DEPRECATED: use `connection_timeout_ms` | +| `experimental_pub_sub_channel` | string | — | (Empty String / No Channel) | An optional and experimental Redis channel to publish write events to. | +| `key_prefix` | string | — | (Empty String / No Prefix) | An optional prefix to prepend to all keys in this store. | +| `mode` | [RedisMode](#redismode) | — | standard, | Set the mode Redis is operating in. | +| `broadcast_channel_capacity` | integer (uint) | — | `0` | Deprecated as redis-rs doesn't use it | +| `command_timeout_ms` | integer (uint64) | — | 10000 (10 seconds) | The amount of time in milliseconds until the Redis store considers the command to be timed out. This will trigger a retry of the command and potentially a reconnection to the Redis server. | +| `connection_timeout_ms` | integer (uint64) | — | 3000 (3 seconds) | The amount of time in milliseconds until the Redis store considers the connection to unresponsive. This will trigger a reconnection to the Redis server. | +| `health_check_timeout_ms` | integer (uint64) | — | 4000 (4 seconds) | Per-call ceiling for the `check_health` PING in milliseconds. | +| `read_chunk_size` | integer (uint) | — | 64KiB | The amount of data to read from the Redis server at a time. This is used to limit the amount of memory used when reading large objects from the Redis server as well as limiting the amount of time a single read operation can take. | +| `connection_pool_size` | integer (uint) | — | 3 | The number of connections to keep open to the Redis servers. | +| `max_chunk_uploads_per_update` | integer (uint) | — | 10 | The maximum number of upload chunks to allow per update. This is used to limit the amount of memory used when uploading large objects to the Redis server. A good rule of thumb is to think of the data as: `AVAIL_MEMORY / (read_chunk_size * max_chunk_uploads_per_update) = THORETICAL_MAX_CONCURRENT_UPLOADS` (note: it's a good idea to divide `AVAIL_MAX_MEMORY` by ~10 to account for other memory usage) | +| `scan_count` | integer (uint) | — | 10000 | The COUNT value passed when scanning keys in Redis. This is used to hint the amount of work that should be done per response. | +| `retry` | [Retry](#retry) | — | — | Retry configuration to use when a network request fails. | +| `max_client_permits` | integer (uint) | — | 500 | Maximum number of permitted actions to the Redis store at any one time This stops problems with timeouts due to many, many inflight actions | +| `max_count_per_cursor` | integer (uint64) | — | 1500 | Maximum number of items returned per cursor for the search indexes May reduce thundering herd issues with worker provisioner at higher node counts, | + +## NoopSpec + +_No fields._ + +## ExperimentalMongoSpec + +Configuration for `ExperimentalMongoDB` store. + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `connection_string` | string | Yes | — | `ExperimentalMongoDB` connection string. Example: <mongodb://localhost:27017> or <mongodb+srv://cluster.mongodb.net> | +| `database` | string | — | "nativelink" | The database name to use. | +| `cas_collection` | string | — | "cas" | The collection name for CAS data. | +| `scheduler_collection` | string | — | "scheduler" | The collection name for scheduler data. | +| `key_prefix` | string | — | "" | Prefix to prepend to all keys stored in `MongoDB`. | +| `read_chunk_size` | integer (uint) | — | 65536 (64KB) | The maximum amount of data to read from `MongoDB` in a single chunk (in bytes). | +| `max_concurrent_uploads` | integer (uint) | — | 10 | Deprecated, unused Maximum number of concurrent uploads allowed. | +| `connection_timeout_ms` | integer (uint64) | — | 3000 | Connection timeout in milliseconds. | +| `command_timeout_ms` | integer (uint64) | — | 10000 | Command timeout in milliseconds. | +| `enable_change_streams` | boolean | — | false | Enable `MongoDB` change streams for real-time updates. Required for scheduler subscriptions. | +| `write_concern_w` | string | — | — | Write concern 'w' parameter. Can be a number (e.g., 1) or string (e.g., "majority"). | +| `write_concern_j` | boolean | — | — | Write concern 'j' parameter (journal acknowledgment). | +| `write_concern_timeout_ms` | integer (uint32) | — | — | Write concern timeout in milliseconds. | +| `max_requests` | integer (uint) | — | Unlimited | Limits the number of requests at any one time | + +## LocalWorkerConfig + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `name` | string | — | {Index position in the workers list} | Name of the worker. This is give a more friendly name to a worker for logging and metric publishing. This is also the prefix of the worker id (i.e. "{name}{uuidv6}"). | +| `worker_api_endpoint` | [EndpointConfig](#endpointconfig) | Yes | — | Endpoint which the worker will connect to the scheduler's `WorkerApiService`. | +| `max_action_timeout_s` | integer (uint) | — | 20 minutes | The maximum time an action is allowed to run. If a task requests for a timeout longer than this time limit, the task will be rejected. Value in seconds. | +| `max_upload_timeout_s` | integer (uint) | — | 10 minutes | Maximum time allowed for uploading action results to CAS after execution completes. If upload takes longer than this, the action fails with `DeadlineExceeded` and may be retried by the scheduler. Value in seconds. | +| `max_cleanup_wait_s` | integer (uint) | — | 30 seconds | Maximum time to wait for action directory cleanup before timing out. Value in seconds. | +| `max_cleanup_backoff_ms` | integer (uint) | — | 500 milliseconds | Maximum backoff duration for exponential backoff when waiting for cleanup. Value in milliseconds. | +| `max_inflight_tasks` | integer (uint64) | — | 0 (infinite tasks) | Maximum number of inflight tasks this worker can cope with. | +| `timeout_handled_externally` | boolean | — | false (`NativeLink` fully handles timeouts) | If timeout is handled in `entrypoint` or another wrapper script. If set to true `NativeLink` will not honor the timeout the action requested and instead will always force kill the action after `max_action_timeout` has been reached. If this is set to false, the smaller value of the action's timeout and `max_action_timeout` will be used to which `NativeLink` will kill the action. | +| `entrypoint` | string | — | {Use the command from the job request} | The command to execute on every execution request. This will be parsed as a command + arguments (not shell). Example: "run.sh" and a job with command: "sleep 5" will result in a command like: "run.sh sleep 5". | +| `experimental_precondition_script` | string | — | — | An optional script to run before every action is processed on the worker. The value should be the full path to the script to execute and will pause all actions on the worker if it returns an exit code other than 0. If not set, then the worker will never pause and will continue to accept jobs according to the scheduler configuration. This is useful, for example, if the worker should not take any more actions until there is enough resource available on the machine to handle them. | +| `cas_fast_slow_store` | string | Yes | — | Underlying CAS store that the worker will use to download CAS artifacts. This store must be a `FastSlowStore`. The `fast` store must be a `FileSystemStore` because it will use hardlinks when building out the files instead of copying the files. The slow store must eventually resolve to the same store the scheduler/client uses to send job requests. | +| `upload_action_result` | [UploadActionResultConfig](#uploadactionresultconfig) | — | — | Configuration for uploading action results. | +| `work_directory` | string | Yes | — | The directory work jobs will be executed from. This directory will be fully managed by the worker service and will be purged on startup. This directory and the directory referenced in `local_filesystem_store_ref`'s `stores::FilesystemStore::content_path` must be on the same filesystem. Hardlinks will be used when placing files that are accessible to the jobs that are sourced from `local_filesystem_store_ref`'s `content_path`. | +| `platform_properties` | map of string to [WorkerProperty](#workerproperty) | Yes | — | Properties of this worker. This configuration will be sent to the scheduler and used to tell the scheduler to restrict what should be executed on this worker. | +| `additional_environment` | map of string to [EnvironmentSource](#environmentsource) | — | — | An optional mapping of environment names to set for the execution as well as those specified in the action itself. If set, will set each key as an environment variable before executing the job with the value of the environment variable being the value of the property of the action being executed of that name or the fixed value. | +| `directory_cache` | [DirectoryCacheConfig](#directorycacheconfig) | — | — | Optional directory cache configuration for improving performance by caching reconstructed input directories and using hardlinks instead of rebuilding them from CAS for every action. | +| `use_namespaces` | boolean | — | False | Whether to use namespaces to isolate the execution. This is only available on Linux. It is highly recommended as it avoids a number of issues with zombie processes and also provides additional hermeticity. If explicitly set to true and it is not supported the worker will exit with an error. | +| `use_mount_namespace` | boolean | — | False | Whether to use a mount namespace to isolate the worker root. This is only available on Linux and when `use_namespaces` is true. It is highly recommended provides additional hermeticity. If explicitly set to true and it is not supported or `use_namespaces` is not set to true the worker will exit with an error. | + +## SimpleSpec + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `supported_platform_properties` | map of string to [PropertyType](#propertytype) | — | — | A list of supported platform properties mapped to how these properties are used when the scheduler looks for worker nodes capable of running the task. | +| `retain_completed_for_s` | integer (uint32) | — | 60 seconds | The amount of time to retain completed actions for in case a `WaitExecution` is called after the action has completed. | +| `client_action_timeout_s` | integer (uint64) | — | 60 seconds | Mark operations as completed with error if no client has updated them within this duration. | +| `worker_timeout_s` | integer (uint64) | — | 5 seconds | Remove workers from pool once the worker has not responded in this amount of time in seconds. | +| `max_action_executing_timeout_s` | integer (uint64) | — | 0 (disabled) | Maximum time (seconds) an action can stay in Executing state without any worker update before being timed out and re-queued. This applies regardless of worker keepalive status, catching cases where a worker is alive (sending keepalives) but stuck on a specific action. Set to 0 to disable (relies only on `worker_timeout_s`). | +| `max_job_retries` | integer (uint) | — | 3 | If a job returns an internal error or times out this many times when attempting to run on a worker the scheduler will return the last error to the client. Jobs will be retried and this configuration is to help prevent one rogue job from infinitely retrying and taking up a lot of resources when the task itself is the one causing the server to go into a bad state. | +| `allocation_strategy` | [WorkerAllocationStrategy](#workerallocationstrategy) | — | `"least_recently_used"` | The strategy used to assign workers jobs. | +| `experimental_backend` | [ExperimentalSimpleSchedulerBackend](#experimentalsimpleschedulerbackend) | — | memory | The storage backend to use for the scheduler. | +| `worker_match_logging_interval_s` | integer (int64) | — | `10` | Every N seconds, do logging of worker matching e.g. "worker busy", "can't find any worker" Defaults to 10s. Can be set to `-1` to disable | +| `fallback_match_interval_s` | integer (int64) | — | `5` | Every N seconds, run a worker matching pass even if no task or worker change notification arrived. This is a safety net for missed notifications and for scheduler backends with eventually consistent searches (for example Redis), where an operation that was re-queued may not be visible to the search triggered by its own notification. Without this, such an operation can stay queued until an unrelated event triggers another matching pass. Defaults to 5s. Zero or any negative value disables it. | + +## SchedulerGrpcSpec + +A scheduler that forwards requests to an upstream scheduler. This +is useful to use when doing some kind of local action cache or CAS away from +the main cluster of workers. In general, it's more efficient to point the +build at the main scheduler directly though. + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `endpoint` | [GrpcEndpoint](#grpcendpoint) | Yes | — | The upstream scheduler to forward requests to. | +| `retry` | [Retry](#retry) | — | — | Retry configuration to use when a network request fails. | +| `max_concurrent_requests` | integer (uint) | — | unlimited | Limit the number of simultaneous upstream requests to this many. A value of zero is treated as unlimited. If the limit is reached the request is queued. | +| `connections_per_endpoint` | integer (uint) | — | 1 | The number of connections to make to each specified endpoint to balance the load over multiple TCP connections. | + +## CacheLookupSpec + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `ac_store` | string | Yes | — | The reference to the action cache store used to return cached actions from rather than running them again. To prevent unintended issues, this store should probably be a `CompletenessCheckingSpec`. | +| `scheduler` | [SchedulerSpec](#schedulerspec) | Yes | — | The nested scheduler to use if cache lookup fails. | + +## PropertyModifierSpec + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `modifications` | array of [PropertyModification](#propertymodification) | Yes | — | A list of modifications to perform to incoming actions for the nested scheduler. These are performed in order and blindly, so removing a property that doesn't exist is fine and overwriting an existing property is also fine. If adding properties that do not exist in the nested scheduler is not supported and will likely cause unexpected behaviour. | +| `scheduler` | [SchedulerSpec](#schedulerspec) | Yes | — | The nested scheduler to use after modifying the properties. | + +## HistoricalResourceSpec + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `hints_file` | string | Yes | — | JSON file containing historical resource hints keyed by Bazel `RequestMetadata` `target_id` and/or `action_mnemonic`. | +| `refresh_interval_s` | integer (uint64) | — | 30 seconds | Reload interval for `hints_file`. Set to 0 to load once. | +| `cpu_property_name` | string | — | `cpu_count` | Platform property name used for CPU minimum values. | +| `memory_property_name` | string | — | `memory_kb` | Platform property name used for memory minimum values, expressed in KiB. | +| `scheduler` | [SchedulerSpec](#schedulerspec) | Yes | — | The nested scheduler to use after applying resource hints. | + +## ListenerConfig + +Plus exactly one of the following variants (the key selects the variant): + +### `http` + +Listener for HTTP/HTTPS/HTTP2 sockets. + +**Type:** [HttpListener](#httplistener) + +## ServicesConfig + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `cas` | array of [CasServiceConfig](#casserviceconfig) | — | — | The Content Addressable Storage (CAS) backend config. The key is the `instance_name` used in the protocol and the value is the underlying CAS store config. | +| `ac` | array of [ActionCacheServiceConfig](#actioncacheserviceconfig) | — | — | The Action Cache (AC) backend config. The key is the `instance_name` used in the protocol and the value is the underlying AC store config. | +| `capabilities` | array of [CapabilitiesServiceConfig](#capabilitiesserviceconfig) | — | — | Capabilities service is required in order to use most of the bazel protocol. This service is used to provide the supported features and versions of this bazel gRPC service. | +| `execution` | array of [ExecutionServiceConfig](#executionserviceconfig) | — | — | The remote execution service configuration. NOTE: This service is under development and is currently just a place holder. | +| `bytestream` | array of [ByteStreamServiceConfig](#bytestreamserviceconfig) | — | — | This is the service used to stream data to and from the CAS. Bazel's protocol strongly encourages users to use this streaming interface to interact with the CAS when the data is large. | +| `fetch` | array of [FetchServiceConfig](#fetchserviceconfig) | — | — | These two are collectively the Remote Asset protocol, but it's defined as two separate services | +| `push` | array of [PushServiceConfig](#pushserviceconfig) | — | — | | +| `worker_api` | [WorkerApiConfig](#workerapiconfig) | — | — | This is the service used for workers to connect and communicate through. NOTE: This service should be served on a different, non-public port. In other words, `worker_api` configuration should not have any other services that are served on the same port. Doing so is a security risk, as workers have a different permission set than a client that makes the remote execution/cache requests. | +| `experimental_bep` | [BepConfig](#bepconfig) | — | — | Experimental - Build Event Protocol (BEP) configuration. This is the service that will consume build events from the client and publish them to a store for processing by an external service. | +| `admin` | [AdminConfig](#adminconfig) | — | — | This is the service for any administrative tasks. It provides a REST API endpoint for administrative purposes. | +| `health` | [HealthConfig](#healthconfig) | — | — | This is the service for health status check. | + +## IdentityHeaderSpec + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `header_name` | string | — | "x-identity" | The name of the header to look for the identity in. | +| `required` | boolean | — | `false` | If the header is required to be set or fail the request. | + +## OriginEventsPublisherSpec + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `store` | string | Yes | — | The store to publish nativelink events to. The store name referenced in the `stores` map in the main config. | + +## ConfigDigestHashFunction + +| Value | Description | +| --- | --- | +| `"sha256"` | Use the sha256 hash function. [https://en.wikipedia.org/wiki/SHA-2](https://en.wikipedia.org/wiki/SHA-2) | +| `"blake3"` | Use the blake3 hash function. [https://en.wikipedia.org/wiki/BLAKE_(hash_function)](https://en.wikipedia.org/wiki/BLAKE_(hash_function)) | + +## EvictionPolicy + +Eviction policy always works on LRU (Least Recently Used). Any time an entry +is touched it updates the timestamp. Inserts and updates will execute the +eviction policy removing any expired entries and/or the oldest entries +until the store size becomes smaller than `max_bytes`. + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `max_bytes` | integer (uint) | — | 0 | Maximum number of bytes before eviction takes place. Zero means never evict based on size. | +| `evict_bytes` | integer (uint) | — | 0 | When eviction starts based on hitting `max_bytes`, continue until `max_bytes - evict_bytes` is met to create a low watermark. This stops operations from thrashing when the store is close to the limit. | +| `max_seconds` | integer (uint32) | — | 0 | Maximum number of seconds for an entry to live since it was last accessed before it is evicted. Zero means never evict based on time. | +| `max_count` | integer (uint64) | — | 0 | Maximum size of the store before an eviction takes place. Zero means never evict based on count. | + +## Retry + +Retry configuration. This configuration is exponential and each iteration +a jitter as a percentage is applied of the calculated delay. For example: +```haskell +Retry{ + max_retries: 7, + delay: 0.1, + jitter: 0.5, +} +``` +will result in: + +| Attempt | Delay | +| --- | --- | +| 1 | 0 ms | +| 2 | 75 to 125 ms | +| 3 | 150 to 250 ms | +| 4 | 300 to 500 ms | +| 5 | 600 ms to 1 s | +| 6 | 1.2 to 2 s | +| 7 | 2.4 to 4 s | +| 8 | 4.8 to 8 s | + +The total delay is additive, so this example produces 9.525 to 15.875 s of total delay for a single request. + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `max_retries` | integer (uint) | — | `0` | Maximum number of retries until retrying stops. Setting this to zero will always attempt 1 time, but not retry. | +| `delay` | number (float) | — | `0` | Delay in seconds for exponential back off. | +| `jitter` | number (float) | — | `0` | Amount of jitter to add as a percentage in decimal form. This will change the formula like: | +| `retry_on_errors` | array of [ErrorCode](#errorcode) | — | — | A list of error codes to retry on, if this isn't set then the default error codes to retry on are used. These default codes are the most likely to be non-permanent: `Unknown`; `Cancelled`; `DeadlineExceeded`; `ResourceExhausted`; `Aborted`; `Internal`; `Unavailable`; `DataLoss` | + +## ExperimentalOntapS3Spec + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `endpoint` | string | Yes | — | | +| `vserver_name` | string | Yes | — | | +| `bucket` | string | Yes | — | | +| `root_certificates` | string | — | — | | +| `key_prefix` | string | — | — | If you wish to prefix the location in the bucket. If None, no prefix will be used. | +| `retry` | [Retry](#retry) | — | — | Retry configuration to use when a network request fails. | +| `consider_expired_after_s` | integer (uint32) | — | 0 | If the number of seconds since the `last_modified` time of the object is greater than this value, the object will not be considered "existing". This allows for external tools to delete objects that have not been uploaded in a long time. If a client receives a `NotFound` the client should re-upload the object. | +| `max_retry_buffer_per_request` | integer (uint) | — | 5MB | The maximum buffer size to retain in case of a retryable error during upload. Setting this to zero will disable upload buffering; this means that in the event of a failure during upload, the entire upload will be aborted and the client will likely receive an error. | +| `multipart_max_concurrent_uploads` | integer (uint) | — | 10 | Maximum number of concurrent `UploadPart` requests per `MultipartUpload`. | +| `insecure_allow_http` | boolean | — | false | Allow unencrypted HTTP connections. Only use this for local testing. | +| `disable_http2` | boolean | — | false | Disable HTTP/2 connections and only use HTTP/1.1. Default client configuration will have HTTP/1.1 and HTTP/2 enabled for connection schemes. HTTP/2 should be disabled if environments have poor support or performance related to HTTP/2. Safe to keep default unless underlying network environment, S3, or GCS API servers specify otherwise. | + +## CompressionAlgorithm + +Plus exactly one of the following variants (the key selects the variant): + +### `lz4` + +LZ4 compression algorithm is extremely fast for compression and +decompression, however does not perform very well in compression +ratio. In most cases build artifacts are highly compressible, however +lz4 is quite good at aborting early if the data is not deemed very +compressible. + +see: [https://lz4.github.io/lz4/](https://lz4.github.io/lz4/) + +**Type:** [Lz4Config](#lz4config) + +## StoreDirection + +| Value | Description | +| --- | --- | +| `"both"` | The store operates normally and all get and put operations are handled by it. | +| `"update"` | Update operations will cause persistence to this store, but Get operations will be ignored. This only makes sense on the fast store as the slow store will never get written to on Get anyway. | +| `"get"` | Get operations will cause persistence to this store, but Update operations will be ignored. | +| `"read_only"` | Operate as a read only store, only really makes sense if there's another way to write to it. | + +## ShardConfig + +Configuration for an individual shard of the store. + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `store` | [NamedStoreConfig](#namedstoreconfig) | Yes | — | Store to shard the data to. | +| `weight` | integer (uint32) | — | 1 | The weight of the store. This is used to determine how much data should be sent to the store. The actual percentage is the sum of all the store's weights divided by the individual store's weight. | + +## GrpcEndpoint + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `address` | string | Yes | — | The endpoint address (i.e. `grpc(s)://example.com:443`). | +| `tls_config` | [ClientTlsConfig](#clienttlsconfig) | — | — | The TLS configuration to use to connect to the endpoint (if grpcs). | +| `concurrency_limit` | integer (uint) | — | — | The maximum concurrency to allow on this endpoint. | +| `connect_timeout_s` | integer (uint64) | — | 30 seconds | Timeout for establishing a TCP connection to the endpoint (seconds). | +| `tcp_keepalive_s` | integer (uint64) | — | 30 seconds | TCP keepalive interval (seconds). Sends TCP keepalive probes at this interval to detect dead connections at the OS level. | +| `http2_keepalive_interval_s` | integer (uint64) | — | 30 seconds | HTTP/2 keepalive interval (seconds). Sends HTTP/2 PING frames at this interval to detect dead connections at the application level. | +| `http2_keepalive_timeout_s` | integer (uint64) | — | 20 seconds | HTTP/2 keepalive timeout (seconds). If a PING response is not received within this duration, the connection is considered dead. | + +## StoreType + +| Value | Description | +| --- | --- | +| `"cas"` | The store is content addressable storage. | +| `"ac"` | The store is an action cache. | + +## GrpcReadBatchingConfig + +Configuration for experimental small-blob read coalescing in a gRPC +store. See [`GrpcSpec::experimental_read_batching`]. + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `max_blob_size_bytes` | integer (uint64) | — | 131072 (128 KiB) | Only blobs at or below this size (in bytes) are eligible for batching. Larger blobs always use the `ByteStream` `Read` path. | +| `max_batch_bytes` | integer (uint64) | — | 3145728 (3 MiB) | Maximum total payload bytes packed into a single `BatchReadBlobs` request. This should leave headroom under the 4 MiB default gRPC message limit for protobuf framing overhead. | +| `dispatch_slots` | integer (uint) | — | 4 | Maximum number of concurrent `BatchReadBlobs` RPCs dispatched by the coalescer. Must be greater than zero. | +| `max_queued_bytes` | integer (uint64) | — | 33554432 (32 MiB) | Bound on the number of payload bytes waiting in the coalescer queue. When exceeded, new read requests bypass batching and fall back to the regular `ByteStream` `Read` path instead of blocking. | + +## RedisMode + +| Value | Description | +| --- | --- | +| `"cluster"` | Use Redis Cluster. | +| `"sentinel"` | Use Redis Sentinel. | +| `"standard"` | Use a standalone Redis server. | + +## EndpointConfig + +Generic config for an endpoint and associated configs. + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `uri` | string | Yes | — | URI of the endpoint. | +| `timeout` | number (float) | — | 5 seconds | Timeout in seconds that a request should take. | +| `tls_config` | [ClientTlsConfig](#clienttlsconfig) | — | — | The TLS configuration to use to connect to the endpoint. | + +## UploadActionResultConfig + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `ac_store` | string | — | {No uploading is done} | Underlying AC store that the worker will use to publish execution results into. Objects placed in this store should be reachable from the scheduler/client-cas after they have finished updating. | +| `upload_ac_results_strategy` | [UploadCacheResultsStrategy](#uploadcacheresultsstrategy) | — | `SuccessOnly` | In which situations should the results be published to the `ac_store`, if set to `SuccessOnly` then only results with an exit code of 0 will be uploaded, if set to Everything all completed results will be uploaded. | +| `historical_results_store` | string | — | {CAS store of parent} | Store to upload historical results to. This should be a CAS store if set. | +| `upload_historical_results_strategy` | [UploadCacheResultsStrategy](#uploadcacheresultsstrategy) | — | `FailuresOnly` | In which situations should the results be published to the historical CAS. The historical CAS is where failures are published. These messages conform to the CAS key-value lookup format and are always a `HistoricalExecuteResponse` serialized message. | +| `success_message_template` | string | — | "" (no message) | Template to use for the `ExecuteResponse.message` property. This message is attached to the response before it is sent to the client. The following special variables are supported:; `digest_function`: Digest function used to calculate the action digest: `action_digest_hash`: Action digest hash: `action_digest_size`: Action digest size: `historical_results_hash`: `HistoricalExecuteResponse` digest hash: `historical_results_size`: `HistoricalExecuteResponse` digest size. | +| `failure_message_template` | string | — | "" (no message) | Same as `success_message_template` but for failure case. | + +## WorkerProperty + +Plus exactly one of the following variants (the key selects the variant): + +### `values` + +List of static values. +Note: Generally there should only ever be 1 value, but if the platform +property key is `PropertyType::Priority` it may have more than one value. + +**Type:** array of string + +### `query_cmd` + +A dynamic configuration. The string will be executed as a command +(not shell) and will be split by "\n" (new line character). + +**Type:** string + +## EnvironmentSource + +One of: + +- `property`: string — The name of the platform property in the action to get the value from. +- `value`: string — The raw value to set. +- `"from_environment"` — Take the value from the local environment corresponding to the name key +- `"timeout_millis"` — The max amount of time in milliseconds the command is allowed to run (requested by the client). +- `"side_channel_file"` — A special file path will be provided that can be used to communicate with the parent process about out-of-band information. This file will be read after the command has finished executing. Based on the contents of the file, the behavior of the result may be modified. +- `"action_directory"` — A "root" directory for the action. This directory can be used to store temporary files that are not needed after the action has completed. This directory will be purged after the action has completed. + +## DirectoryCacheConfig + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `max_entries` | integer (uint) | — | 1000 | Maximum number of cached directories. | +| `max_size_bytes` | integer (uint64) | — | 10737418240 (10 GB) | Maximum total size in bytes for all cached directories (0 = unlimited). | +| `cache_root` | string | — | `{work_directory}/../directory_cache` | Base directory for cache storage. This directory will be managed by the worker and should be on the same filesystem as `work_directory`. | +| `experimental_subtree_caching` | boolean | — | false (only root directories are cached; existing behavior) | Optional and experimental: additionally cache every subdirectory by its own `Directory` digest, not just the root directory. REAPI Merkle nodes are content-addressed, so a subtree that is byte-identical between two different roots has the same digest and can be materialized with a single hardlink pass instead of being rebuilt from the CAS. This makes the common "one input file changed out of thousands" case reuse every unchanged subtree. | +| `max_concurrent_fetches` | integer (uint) | — | 64 | Maximum number of concurrent slow-store fetches across ALL directory constructions of this cache. This bound protects backing stores from RPC storms: per-level construction concurrency compounds multiplicatively across tree levels and concurrent actions. | + +## PropertyType + +When the scheduler matches tasks to workers that are capable of running +the task, this value will be used to determine how the property is treated. + +One of: + +- string +- `"minimum"` — Requires the platform property to be a u64 and when the scheduler looks for appropriate worker nodes that are capable of executing the task, the task will not run on a node that has less than this value. +- `"exact"` — Requires the platform property to be a string and when the scheduler looks for appropriate worker nodes that are capable of executing the task, the task will not run on a node that does not have this property set to the value with exact string match. +- `"priority"` — Does not restrict on this value and instead will be passed to the worker as an informational piece. In the future this will be used by the scheduler and worker to cause the scheduler to prefer certain workers over others, but not restrict them based on these values. + +## WorkerAllocationStrategy + +When a worker is being searched for to run a job, this will be used +on how to choose which worker should run the job when multiple +workers are able to run the task. + +| Value | Description | +| --- | --- | +| `"least_recently_used"` | Prefer workers that have been least recently used to run a job. | +| `"most_recently_used"` | Prefer workers that have been most recently used to run a job. | + +## ExperimentalSimpleSchedulerBackend + +One of: + +- `"memory"` — Use an in-memory store for the scheduler. +- `redis`: [ExperimentalRedisSchedulerBackend](#experimentalredisschedulerbackend) — Use a Redis store for the scheduler. + +## SchedulerSpec + +Plus exactly one of the following variants (the key selects the variant): + +### `simple` + +**Type:** [SimpleSpec](#simplespec) + +### `grpc` + +**Type:** [SchedulerGrpcSpec](#schedulergrpcspec) + +### `cache_lookup` + +**Type:** [CacheLookupSpec](#cachelookupspec) + +### `property_modifier` + +**Type:** [PropertyModifierSpec](#propertymodifierspec) + +### `historical_resource` + +**Type:** [HistoricalResourceSpec](#historicalresourcespec) + +## PropertyModification + +Plus exactly one of the following variants (the key selects the variant): + +### `add` + +Add a property to the action properties. + +**Type:** [PlatformPropertyAddition](#platformpropertyaddition) + +### `remove` + +Remove a named property from the action. + +**Type:** string + +### `replace` + +If a property is found, then replace it with another one. + +**Type:** [PlatformPropertyReplacement](#platformpropertyreplacement) + +## HttpListener + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `socket_address` | string | Yes | — | Address to listen on. Example: `127.0.0.1:8080` or `:8080` to listen to all IPs. | +| `freebind` | boolean | — | false | Allow binding `socket_address` before it is assigned locally. | +| `compression` | [HttpCompressionConfig](#httpcompressionconfig) | — | — | Data transport compression configuration to use for this service. | +| `advanced_http` | [HttpServerConfig](#httpserverconfig) | — | — | Advanced HTTP server configuration. | +| `max_decoding_message_size` | integer (uint) | — | 4 MiB | Maximum number of bytes to decode on each grpc stream chunk. | +| `tls` | [TlsConfig](#tlsconfig) | — | — | TLS configuration for this server. If not set, the server will not use TLS. | + +## CasServiceConfig + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `instance_name` | string | — | `""` | Used when the config references `instance_name` in the protocol. | +| `cas_store` | string | Yes | — | The store name referenced in the `stores` map in the main config. This store name referenced here may be reused multiple times. | +| `experimental_chunking` | [CasChunkingConfig](#caschunkingconfig) | — | not set — chunking RPCs are rejected, nothing is advertised, | Optional and experimental: enables the REAPI `SplitBlob`/`SpliceBlob` RPCs used by content-defined chunking clients (e.g. Bazel's `--experimental_remote_cache_chunking`). When set, the capabilities service advertises blob split/splice support and `FastCDC` 2020 parameters for this instance. When `cas_store` is a grpc store the RPCs are forwarded to the backend (which must support chunking with matching parameters); otherwise they are served locally. | + +## ActionCacheServiceConfig + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `instance_name` | string | — | `""` | Used when the config references `instance_name` in the protocol. | +| `ac_store` | string | Yes | — | The store name referenced in the `stores` map in the main config. This store name referenced here may be reused multiple times. | +| `read_only` | boolean | — | `false` | Whether the Action Cache store may be written to, this if set to false it is only possible to read from the Action Cache. | + +## CapabilitiesServiceConfig + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `instance_name` | string | — | `""` | Used when the config references `instance_name` in the protocol. | +| `remote_execution` | [CapabilitiesRemoteExecutionConfig](#capabilitiesremoteexecutionconfig) | — | — | Configuration for remote execution capabilities. If not set the capabilities service will inform the client that remote execution is not supported. | +| `remote_cache_compression` | boolean | — | — | Whether this instance supports Bazel remote cache compression. When enabled, the capabilities service advertises zstd wire compression and the ByteStream/CAS services accept REAPI compressed-blobs/zstd data. | + +## ExecutionServiceConfig + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `instance_name` | string | — | `""` | Used when the config references `instance_name` in the protocol. | +| `cas_store` | string | Yes | — | The store name referenced in the `stores` map in the main config. This store name referenced here may be reused multiple times. This value must be a CAS store reference. | +| `scheduler` | string | Yes | — | The scheduler name referenced in the `schedulers` map in the main config. | + +## ByteStreamServiceConfig + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `instance_name` | string | — | `""` | Used when the config references `instance_name` in the protocol. | +| `cas_store` | string | Yes | — | Name of the store in the "stores" configuration. | +| `max_bytes_per_stream` | integer (uint) | — | 64KiB | Max number of bytes to send on each grpc stream chunk. According to [https://github.com/grpc/grpc.github.io/issues/371](https://github.com/grpc/grpc.github.io/issues/371) 16KiB - 64KiB is optimal. | +| `persist_stream_on_disconnect_timeout_s` | integer (uint) | — | 10 seconds | In the event a client disconnects while uploading a blob, we will hold the internal stream open for this many seconds before closing it. This allows clients that disconnect to reconnect and continue uploading the same blob. | + +## FetchServiceConfig + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `instance_name` | string | — | `""` | Used when the config references `instance_name` in the protocol. | +| `fetch_store` | string | Yes | — | The store name referenced in the `stores` map in the main config. This store name referenced here may be reused multiple times. | + +## PushServiceConfig + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `instance_name` | string | — | `""` | Used when the config references `instance_name` in the protocol. | +| `push_store` | string | Yes | — | The store name referenced in the `stores` map in the main config. This store name referenced here may be reused multiple times. | +| `read_only` | boolean | — | `false` | Whether the Action Cache store may be written to, this if set to false it is only possible to read from the Action Cache. | + +## WorkerApiConfig + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `scheduler` | string | Yes | — | The scheduler name referenced in the `schedulers` map in the main config. | + +## BepConfig + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `store` | string | Yes | — | The store to publish build events to. The store name referenced in the `stores` map in the main config. | + +## AdminConfig + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `path` | string | — | "/admin" | Path to register the admin API. If path is "/admin", and your domain is "example.com", you can reach the endpoint with: [http://example.com/admin](http://example.com/admin). | + +## HealthConfig + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `path` | string | — | "/status" | Path to register the health status check. If path is "/status", and your domain is "example.com", you can reach the endpoint with: [http://example.com/status](http://example.com/status). | +| `timeout_seconds` | integer (uint64) | — | 5s | Timeout on health checks. | + +## ErrorCode + +The possible error codes that might occur on an upstream request. + +| Value | Description | +| --- | --- | +| `"Cancelled"` | | +| `"Unknown"` | | +| `"InvalidArgument"` | | +| `"DeadlineExceeded"` | | +| `"NotFound"` | | +| `"AlreadyExists"` | | +| `"PermissionDenied"` | | +| `"ResourceExhausted"` | | +| `"FailedPrecondition"` | | +| `"Aborted"` | | +| `"OutOfRange"` | | +| `"Unimplemented"` | | +| `"Internal"` | | +| `"Unavailable"` | | +| `"DataLoss"` | | +| `"Unauthenticated"` | | + +## Lz4Config + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `block_size` | integer (uint32) | — | 65536 (64k) | Size of the blocks to compress. Higher values require more ram, but might yield slightly better compression ratios. | +| `max_decode_block_size` | integer (uint32) | — | value in `block_size` | Maximum size allowed to attempt to deserialize data into. This is needed because the `block_size` is embedded into the data so if there was a bad actor, they could upload an extremely large `block_size`'ed entry and we'd allocate a large amount of memory when retrieving the data. To prevent this from happening, we allow you to specify the maximum that we'll attempt to deserialize. | + +## ClientTlsConfig + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `ca_file` | string | — | — | Path to the certificate authority to use to validate the remote. | +| `cert_file` | string | — | — | Path to the certificate file for client authentication. | +| `key_file` | string | — | — | Path to the private key file for client authentication. | +| `use_native_roots` | boolean | — | false | If set the client will use the native roots for TLS connections. | + +## UploadCacheResultsStrategy + +| Value | Description | +| --- | --- | +| `"success_only"` | Only upload action results with an exit code of 0. | +| `"never"` | Don't upload any action results. | +| `"everything"` | Upload all action results that complete. | +| `"failures_only"` | Only upload action results that fail. | + +## ExperimentalRedisSchedulerBackend + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `redis_store` | string | Yes | — | A reference to the Redis store to use for the scheduler. Note: This MUST resolve to a `RedisSpec`. | + +## PlatformPropertyAddition + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `name` | string | Yes | — | The name of the property to add. | +| `value` | string | Yes | — | The value to assign to the property. | + +## PlatformPropertyReplacement + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `name` | string | Yes | — | The name of the property to replace. | +| `value` | string | — | — | The value to match against, if unset then any instance matches. | +| `new_name` | string | Yes | — | The new name of the property. | +| `new_value` | string | — | — | The value to assign to the property, if unset will remain the same. | + +## HttpCompressionConfig + +Note: Compressing data in the cloud rarely has a benefit, since most +cloud providers have very high bandwidth backplanes. However, for +clients not inside the data center, it might be a good idea to +compress data to and from the cloud. This will however come at a high +CPU and performance cost. If you are making remote execution share the +same CAS/AC servers as client's remote cache, you can create multiple +services with different compression settings that are served on +different ports. Then configure the non-cloud clients to use one port +and cloud-clients to use another. + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `send_compression_algorithm` | [HttpCompressionAlgorithm](#httpcompressionalgorithm) | — | `HttpCompressionAlgorithm::None` | The compression algorithm that the server will use when sending responses to clients. Enabling this will likely save a lot of data transfer, but will consume a lot of CPU and add a lot of latency. see: [https://github.com/tracemachina/nativelink/issues/109](https://github.com/tracemachina/nativelink/issues/109) | +| `accepted_compression_algorithms` | array of [HttpCompressionAlgorithm](#httpcompressionalgorithm) | Yes | {no supported compression} | The compression algorithm that the server will accept from clients. The server will broadcast the supported compression algorithms to clients and the client will choose which compression algorithm to use. Enabling this will likely save a lot of data transfer, but will consume a lot of CPU and add a lot of latency. see: [https://github.com/tracemachina/nativelink/issues/109](https://github.com/tracemachina/nativelink/issues/109) | + +## HttpServerConfig + +Advanced HTTP configuration. These generally should not be set. +For documentation on these settings, see the hyper documentation: +See: [hyper HTTP server docs](https://docs.rs/hyper/latest/hyper/server/conn/struct.Http.html) + +Note: All of these default to the default values from hyper unless otherwise +specified. + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `http2_keep_alive_interval` | integer (uint32) | — | — | Interval to send keep-alive pings via HTTP2. Note: This is in seconds. | +| `experimental_http2_max_pending_accept_reset_streams` | integer (uint32) | — | — | | +| `experimental_http2_initial_stream_window_size` | integer (uint32) | — | — | | +| `experimental_http2_initial_connection_window_size` | integer (uint32) | — | — | | +| `experimental_http2_adaptive_window` | boolean | — | — | | +| `experimental_http2_max_frame_size` | integer (uint32) | — | — | | +| `experimental_http2_max_concurrent_streams` | integer (uint32) | — | — | | +| `experimental_http2_keep_alive_timeout_s` | integer (uint32) | — | — | | +| `experimental_http2_max_send_buf_size` | integer (uint32) | — | — | | +| `experimental_http2_enable_connect_protocol` | boolean | — | — | | +| `experimental_http2_max_header_list_size` | integer (uint32) | — | — | | + +## TlsConfig + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `cert_file` | string | Yes | — | Path to the certificate file. | +| `key_file` | string | Yes | — | Path to the private key file. | +| `client_ca_file` | string | — | — | Path to the certificate authority for mTLS, if client authentication is required for this endpoint. | +| `client_crl_file` | string | — | — | Path to the certificate revocation list for mTLS, if client authentication is required for this endpoint. | + +## CasChunkingConfig + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `index_store` | string | — | — | The store name referenced in the `stores` map in the main config used to persist blob-to-chunks layouts. Keys are the digests of the original blobs and values are serialized chunk layouts (which do not hash to those digests), so this store MUST NOT perform content digest verification and MUST NOT be the same store as `cas_store` — writing layouts into the CAS would overwrite blob content. Using the same store name as `cas_store` is rejected at startup. | +| `avg_chunk_size_bytes` | integer (uint64) | — | 524288 (512 KiB) | The average chunk size in bytes advertised to clients through the `FastCDC` 2020 capability parameters and used for server-side chunking in `SplitBlob`. Clients derive the minimum and maximum chunk sizes from this value (avg / 4 and avg * 4). The value must be between 1 KiB and 1 MiB. | +| `max_chunk_count` | integer (uint64) | — | 50000 | Maximum number of chunks accepted in a `SpliceBlob` request or produced by on-demand chunking in `SplitBlob`. Blobs that would produce more chunks are served without chunking (`SplitBlob` returns `NOT_FOUND` and clients fall back to a regular download). This bounds the size of stored chunk layouts and of `SplitBlobResponse` messages (roughly 80-140 bytes per chunk). At the default average chunk size the default cap supports blobs up to ~25 GiB; note that values above ~50000 may produce responses that exceed default gRPC message size limits on clients. | + +## CapabilitiesRemoteExecutionConfig + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `scheduler` | string | Yes | — | Scheduler used to configure the capabilities of remote execution. | + +## HttpCompressionAlgorithm + +| Value | Description | +| --- | --- | +| `"none"` | No compression. | +| `"gzip"` | Zlib compression. | + +## Reading the source + +If anything here disagrees with the binary, the source wins: + +- [`stores.rs`](https://github.com/TraceMachina/nativelink/tree/v1.6.2/nativelink-config/src/stores.rs) +- [`cas_server.rs`](https://github.com/TraceMachina/nativelink/tree/v1.6.2/nativelink-config/src/cas_server.rs) +- [`schedulers.rs`](https://github.com/TraceMachina/nativelink/tree/v1.6.2/nativelink-config/src/schedulers.rs) diff --git a/web/apps/docs/lib/config-versions.ts b/web/apps/docs/lib/config-versions.ts index 624a044b4..e65a5cec0 100644 --- a/web/apps/docs/lib/config-versions.ts +++ b/web/apps/docs/lib/config-versions.ts @@ -28,13 +28,45 @@ export const CONFIG_VERSIONS: ConfigVersion[] = [ "isDev": true }, { - "version": "v1.5.2", - "label": "v1.5.2 (latest)", + "version": "v1.6.3", + "label": "v1.6.3 (latest)", "href": "/reference/nativelink-config", - "ref": "v1.5.2", + "ref": "v1.6.3", "isLatest": true, "isDev": false }, + { + "version": "v1.6.2", + "label": "v1.6.2", + "href": "/reference/nativelink-config/v1.6.2", + "ref": "v1.6.2", + "isLatest": false, + "isDev": false + }, + { + "version": "v1.6.1", + "label": "v1.6.1", + "href": "/reference/nativelink-config/v1.6.1", + "ref": "v1.6.1", + "isLatest": false, + "isDev": false + }, + { + "version": "v1.6.0", + "label": "v1.6.0", + "href": "/reference/nativelink-config/v1.6.0", + "ref": "v1.6.0", + "isLatest": false, + "isDev": false + }, + { + "version": "v1.5.2", + "label": "v1.5.2", + "href": "/reference/nativelink-config/v1.5.2", + "ref": "v1.5.2", + "isLatest": false, + "isDev": false + }, { "version": "v1.5.1", "label": "v1.5.1", @@ -99,14 +131,6 @@ export const CONFIG_VERSIONS: ConfigVersion[] = [ "isLatest": false, "isDev": false }, - { - "version": "v1.0.1", - "label": "v1.0.1", - "href": "/reference/nativelink-config/v1.0.1", - "ref": "v1.0.1", - "isLatest": false, - "isDev": false - }, { "version": "v1.0.0", "label": "v1.0.0", diff --git a/web/apps/docs/mdx-components.tsx b/web/apps/docs/mdx-components.tsx index 39873cc62..1c48b6e67 100644 --- a/web/apps/docs/mdx-components.tsx +++ b/web/apps/docs/mdx-components.tsx @@ -2,6 +2,7 @@ import { Callout } from "@/components/callout"; import { ConfigVersionSwitcher } from "@/components/config-version-switcher"; import { Mermaid } from "@/components/mermaid"; import { Steps } from "@/components/steps"; +import { Accordion, Accordions } from "fumadocs-ui/components/accordion"; import { Callout as FumadocsCallout } from "fumadocs-ui/components/callout"; import { Tab, Tabs } from "fumadocs-ui/components/tabs"; import defaultMdxComponents from "fumadocs-ui/mdx"; @@ -20,6 +21,8 @@ export function getMDXComponents(components?: MDXComponents): MDXComponents { Tabs, Tab, Mermaid, + Accordions, + Accordion, ...components, }; } diff --git a/web/apps/docs/next.config.mjs b/web/apps/docs/next.config.mjs index 2fa43e350..27918d7c2 100644 --- a/web/apps/docs/next.config.mjs +++ b/web/apps/docs/next.config.mjs @@ -20,6 +20,30 @@ const nextConfig = { destination: "/getting-started/other-build-systems/siso", permanent: true, }, + // The standalone /faq section was dissolved into per-page FAQ + // sections; keep its published URLs resolving to the new homes. + ...Object.entries({ + architecture: "/explanations/architecture#faq", + caching: "/getting-started/setup#faq", + clients: "/getting-started/other-build-systems#faq", + configuration: "/configuration/intro#faq", + contributing: "/contribute/guidelines#faq", + cost: "/getting-started/on-prem#faq", + deployment: "/deployment/on-prem-overview#faq", + hermeticity: "/explanations/lre#faq", + lre: "/explanations/lre#faq", + nix: "/contribute/nix#faq", + observability: "/deployment/metrics#faq", + "remote-execution": "/rbe/examples#faq", + rust: "/explanations/history#faq", + stores: "/reference/nativelink-config/store-overview#faq", + toolchains: "/rbe/examples#faq", + troubleshooting: "/getting-started/setup#faq", + }).map(([slug, destination]) => ({ + source: `/faq/${slug}`, + destination, + permanent: true, + })), ]; }, }; diff --git a/web/apps/docs/scripts/gen-config-reference.mjs b/web/apps/docs/scripts/gen-config-reference.mjs index 051cc4859..610fb256a 100644 --- a/web/apps/docs/scripts/gen-config-reference.mjs +++ b/web/apps/docs/scripts/gen-config-reference.mjs @@ -26,7 +26,7 @@ // the worktrees don't each spend disk on a full target/ tree. import { execFileSync } from "node:child_process"; -import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync, existsSync } from "node:fs"; import { tmpdir } from "node:os"; import { dirname, join } from "node:path"; import { fileURLToPath } from "node:url"; @@ -193,6 +193,17 @@ function generateOne(v, sharedTargetDir) { sourceDir = worktree; } + const outfile = outFileFor(v); + let existingMdx = ""; + if (existsSync(outfile)) { + existingMdx = readFileSync(outfile, { encoding: 'utf8', flag: 'r' }); + } + const existingVersionMatches = existingMdx.match(/Source: nativelink-config @ (.+)/); + var existingVersion = ""; + if (existingVersionMatches !== null) { + existingVersion = existingVersionMatches[1]; + } + try { const schema = buildSchema(sourceDir, v.isDev ? null : sharedTargetDir); const mdx = schemaToMdx(schema, { @@ -203,7 +214,14 @@ function generateOne(v, sharedTargetDir) { switcher: "", githubBase: GITHUB_BASE, }); - writeFileSync(outFileFor(v), mdx); + const newVersion = mdx.match(/Source: nativelink-config @ (.+)/)[1] + + if (mdx.replace(newVersion, existingVersion) != existingMdx) { + console.log(`Diff between '${existingVersion}' and '${newVersion}', so writing file`); + writeFileSync(outfile, mdx); + } else { + console.log(`No diff between '${existingVersion}' and '${newVersion}', so not writing file`); + } return { ...v, commit, defs: Object.keys(schema.$defs ?? schema.definitions ?? {}).length }; } finally { if (worktree) { diff --git a/web/apps/docs/scripts/lib/schema-to-mdx.mjs b/web/apps/docs/scripts/lib/schema-to-mdx.mjs index 8b935cfd8..38f74f26d 100644 --- a/web/apps/docs/scripts/lib/schema-to-mdx.mjs +++ b/web/apps/docs/scripts/lib/schema-to-mdx.mjs @@ -87,6 +87,11 @@ function stripDescriptionDefault(md) { function normalizeMarkdown(md) { return md .replace(/Example JSON Config/g, "Example JSON5 config") + + // Fixed now in stores, but buggy up to 1.6.1 + .replace(/`NetApp` ONTAP S3/, "NetApp ONTAP S3:") + .replace(/`NetApp` ONTAP S3 store/, "NetApp ONTAP S3 store") + .replace(/```json(?=\n)/g, "```json5") .replace( /will result in:\nAttempt - Delay\n1\s+0ms\n2\s+75ms - 125ms\n3\s+150ms - 250ms\n4\s+300ms - 500ms\n5\s+600ms - 1s\n6\s+1\.2s - 2s\n7\s+2\.4s - 4s\n8\s+4\.8s - 8s/g, diff --git a/web/apps/web/app/careers/member-of-technical-staff/page.mdx b/web/apps/web/app/careers/member-of-technical-staff/page.mdx new file mode 100644 index 000000000..7f1e1f795 --- /dev/null +++ b/web/apps/web/app/careers/member-of-technical-staff/page.mdx @@ -0,0 +1,86 @@ +import { Button, Prose, Section } from "@nativelink/ui"; + +export const metadata = { + title: "Member of Technical Staff, London", + description: + "NativeLink is hiring a Member of Technical Staff in London. Build systems, low latency, Rust. In-office, lunch covered.", +}; + +
    +
    + +← All roles + +
    + +
    + + + +# Member of Technical Staff + +**Location:** London, in-office · **Type:** Full-time · **Team:** Engineering + +## About NativeLink + +NativeLink is a high-performance, open source remote build cache and execution platform. We are a small team in London, and we are growing. + +## The role + +As a Member of Technical Staff you own real systems end to end, from the scheduler and worker fleet to the storage layer clients depend on. You will operate large clusters in production, keep them fast under heavy load, and ship improvements to both our infrastructure and the open source project itself. + +## What you'll do + +- Operate and scale large Kubernetes clusters in production. +- Build and harden the services behind remote execution and caching. +- Diagnose performance problems where they actually live, in the kernel, the network, and the storage layer, and fix them. +- Work in the open: land changes in the public NativeLink repository that the whole community runs. + +## What we're looking for + +- Experience operating large clusters in production. +- Strong Kubernetes. +- Strong Go. +- A love of build systems, low latency, and getting to the bottom of hard systems problems. +- An interest in Rust, and experience with Rust or C++. + +## Nice to have + +- Experience in safety-critical environments, aerospace especially. + +## Security and trust responsibilities + +We run infrastructure our customers' builds — and their source code — depend on, so every engineer here shares responsibility for keeping the platform trustworthy. In this role you will: + +- **Security:** follow our security policies and secure development practices, design and review changes with security in mind, and report and help remediate vulnerabilities promptly. +- **Availability:** build and operate production systems to meet our availability commitments, including monitoring, capacity planning, and participating in incident response. +- **Processing integrity:** ensure build, cache, and execution results are processed completely, accurately, and only as authorized. +- **Confidentiality:** protect confidential customer and company information, and access systems and data only as your role requires, following least privilege. +- **Privacy:** handle personal data in line with our privacy policy and applicable data-protection law. + +## What we offer + +- Real ownership: you own systems end to end on a small team, and your work ships in the open to everyone who runs NativeLink. +- An in-office role in a good part of London, near several tube stations, with lunch covered and an unhurried lunch culture. +- A **£15,000** referral bonus: refer an engineer we hire, and once they thrive for three months, it is yours. + +## How to apply + +Our best hires came to the open source project, found something they wanted to fix, and fixed it. Before you reach out, head to the [repository](https://github.com/TraceMachina/nativelink) and find something you like. We will help you get started, and improving our contributor onboarding is itself a much appreciated contribution. + +When you are ready, apply below. + + + +
    + +
    + +

    Note: recruiters will be blocked and marked as spam.

    + +
    +
    diff --git a/web/apps/web/app/careers/page.tsx b/web/apps/web/app/careers/page.tsx new file mode 100644 index 000000000..4c2fdb543 --- /dev/null +++ b/web/apps/web/app/careers/page.tsx @@ -0,0 +1,141 @@ +import { Badge, Button, Eyebrow, Reveal, Section } from "@nativelink/ui"; + +export const metadata = { + title: "Careers", + description: + "Join NativeLink. We are a small team building the open source remote build cache and execution platform, and we are growing.", +}; + +const roles = [ + { + name: "Member of Technical Staff", + location: "London · In-office", + href: "/careers/member-of-technical-staff", + summary: + "Operate large clusters and own real systems end to end. Strong Go and Kubernetes, a love of build systems and low latency, and an interest in Rust. Safety-critical or aerospace experience is a plus, not a requirement.", + }, +]; + +export default function CareersPage() { + return ( + <> + {/* HERO */} +
    +
    +
    + +
    + Careers +

    + We are making a small team{" "} + + bigger + + . +

    +

    + We build the open source remote build cache and execution + platform, and we are growing. If you love build systems, low + latency, and Rust, we would like to meet you. +

    + +
    +     +
    +
    + + {/* OPEN ROLES */} +
    + +
    + Open roles +

    + Come build with us. +

    +

    + Our current openings are below. We are growing, so more roles, in + more places, are on the way. +

    +
    +     + +
    + + {/* HOW WE HIRE */} +
    + +
    + How we hire +

    + Start with the open source project. +

    +
    +     + +
    +

    + Our best hires came to the open source project, found a bug, and + fixed it. That is the strongest signal we know of, and it is the + best way to reach us. Before you message us, head to the + repository and find something you like. We will help you get + started, and improving our contributor onboarding is itself a much + appreciated contribution. +

    +

    + We prioritize a love of build systems, low latency, an interest + in Rust, and experience with Rust or C++. None of it needs to be + on a resume: it can be in your commits. +

    +
    +     + +

    + Note: recruiters will be blocked and marked as spam. +

    +     +
    + + ); +} diff --git a/web/apps/web/app/company/page.tsx b/web/apps/web/app/company/page.tsx index ce512c8ca..3862a8544 100644 --- a/web/apps/web/app/company/page.tsx +++ b/web/apps/web/app/company/page.tsx @@ -41,13 +41,13 @@ const contactCards = [ { title: "General inquiries", body: "Have a question or comment? We'd love to hear from you.", - cta: { label: "Contact us", href: "mailto:contact@nativelink.com" }, + cta: { label: "Contact us", href: "mailto:contact@tracemachina.com" }, variant: "outline" as const, }, { title: "Sales inquiries", body: "Curious if NativeLink fits your team? Our sales engineers can help.", - cta: { label: "Talk to sales", href: "mailto:hello@nativelink.com" }, + cta: { label: "Talk to sales", href: "mailto:contact@tracemachina.com" }, variant: "primary" as const, }, ]; diff --git a/web/apps/web/app/compliance/page.tsx b/web/apps/web/app/compliance/page.tsx deleted file mode 100644 index 8f79e39f6..000000000 --- a/web/apps/web/app/compliance/page.tsx +++ /dev/null @@ -1,262 +0,0 @@ -import { Badge, Button, Eyebrow, Reveal, Section } from "@nativelink/ui"; - -export const metadata = { - title: "Compliance", - description: - "NativeLink's security posture — SOC 2, GDPR, encryption, audit logs, signed artifacts, and more.", -}; - -const certifications = [ - { - name: "SOC 2 Type II", - status: "In progress", - statusTone: "brand" as const, - body: "Audit underway, expected report Q3 2026. We're tracking against the AICPA Trust Services Criteria today.", - }, - { - name: "GDPR", - status: "Compliant", - statusTone: "success" as const, - body: "Data Processing Addendum available on request. EU data residency for Cloud customers.", - }, - { - name: "CCPA", - status: "Compliant", - statusTone: "success" as const, - body: "California residents can request access, deletion, and opt-out at any time.", - }, - { - name: "ISO 27001", - status: "Planned", - statusTone: "default" as const, - body: "Targeting 2027 once our SOC 2 Type II report is published.", - }, -]; - -const controls = [ - { - icon: ( - - ), - title: "Encryption everywhere", - body: "TLS 1.3 in transit. AES-256 at rest. mTLS between every service in the cluster. Customer keys supported on Enterprise.", - }, - { - icon: ( - <> - ), - title: "Single sign-on", - body: "SAML 2.0 and OIDC with Okta, Azure AD, Google Workspace, JumpCloud. SCIM 2.0 user provisioning on Enterprise.", - }, - { - icon: ( - <> - ), - title: "Audit logs", - body: "Every administrative action and every action result is logged, signed, and exportable to your SIEM via webhook or S3.", - }, - { - icon: ( - - ), - title: "Signed artifacts", - body: "Build inputs and outputs are content-addressed and cryptographically signed. Tampering is detectable at the hash level.", - }, - { - icon: ( - <> - ), - title: "Data residency", - body: "US and EU regions on Cloud. Pin your data to a region by contract. Enterprise customers can run fully air-gapped.", - }, - { - icon: ( - - ), - title: "Vulnerability program", - body: "We run continuous SAST/DAST on every PR. Disclose anything at security@nativelink.com — we triage within 24h.", - }, -]; - -const policies = [ - { name: "Information Security Policy", description: "Our internal standards for handling customer and corporate data." }, - { name: "Incident Response Plan", description: "How we detect, escalate, and communicate during an incident." }, - { name: "Sub-processor list", description: "The vendors that process customer data on our behalf." }, - { name: "Data Processing Addendum", description: "GDPR-compliant DPA for European customers." }, -]; - -export default function CompliancePage() { - return ( - <> - {/* HERO */} -
    -
    -
    - -
    - Compliance -

    - Security you can{" "} - - audit, not just trust - - . -

    -

    - NativeLink runs the build farms that ship safety-critical software. - Every byte is content-addressed, every action is logged, and every - control is auditable. -

    - -
    -     -
    -
    - - {/* CERTIFICATIONS */} -
    - -
    - Certifications -

    - Where we are. Where we're going. -

    -
    -     -
    - {certifications.map((c, i) => ( - -
    -
    -

    - {c.name} -

    - {c.status} -
    -

    - {c.body} -

    -
    -     - ))} -
    -
    - - {/* CONTROLS */} -
    - -
    - Controls -

    - How NativeLink protects your code. -

    -
    -     -
    - {controls.map((c, i) => ( - -
    -
    - -
    -

    - {c.title} -

    -

    - {c.body} - {c.title === "Data residency" && ( - <> - {" "} - - Deploy on-prem with our Helm charts → - - - )} -

    -
    -     - ))} -
    -
    - - {/* POLICIES */} -
    - -
    -
    - Policies & reports -

    - Available on request. -

    -

    - Email{" "} - - security@nativelink.com - {" "} - with your NDA — we'll send the relevant report within one business - day. -

    -
    -
    -     - - -
      - {policies.map((p) => ( -
    • -
      -

      - {p.name} -

      -

      {p.description}

      -
      - - Request - -
      • - ))} -
    -     -
    - - {/* RESPONSIBLE DISCLOSURE */} -
    -
    -
    - - Responsible disclosure -

    - Found something? We want to hear about it. -

    -

    - We triage every report within 24 hours, fix high-severity issues - within 7 days, and credit researchers in our advisory. -

    - -     -
    -
    - - ); -} diff --git a/web/apps/web/app/contact/page.tsx b/web/apps/web/app/contact/page.tsx index 4a850fb81..450e6c7cb 100644 --- a/web/apps/web/app/contact/page.tsx +++ b/web/apps/web/app/contact/page.tsx @@ -7,26 +7,26 @@ export const metadata = { const channels = [ { - label: "Sales", - title: "hello@nativelink.com", - body: "Talk to a solutions engineer about your team's build farm.", - href: "mailto:hello@nativelink.com", + label: "General and sales", + title: "contact@tracemachina.com", + body: "Questions, comments, or talk to us about your team's build farm.", + href: "mailto:contact@tracemachina.com", icon: , }, - { - label: "General", - title: "contact@nativelink.com", - body: "Questions, comments, or anything else.", - href: "mailto:contact@nativelink.com", - icon: <>, - }, { label: "Security", - title: "security@nativelink.com", + title: "security@tracemachina.com", body: "Responsible-disclosure reports. We triage within 24 h.", - href: "mailto:security@nativelink.com", + href: "mailto:security@tracemachina.com", icon: , }, + { + label: "Legal", + title: "legal@tracemachina.com", + body: "Contracts, licensing, and legal enquiries.", + href: "mailto:legal@tracemachina.com", + icon: <>, + }, ]; const socialLinks = [ diff --git a/web/apps/web/app/layout.tsx b/web/apps/web/app/layout.tsx index 8207c6625..90cc1dd27 100644 --- a/web/apps/web/app/layout.tsx +++ b/web/apps/web/app/layout.tsx @@ -1,4 +1,5 @@ import { FinalCTA, SiteFooter, SiteHeader, ThemeProvider, themeInitScript } from "@nativelink/ui"; +import { GoogleTagManager } from "@next/third-parties/google"; import { GeistMono } from "geist/font/mono"; import { GeistSans } from "geist/font/sans"; import type { Metadata } from "next"; @@ -27,6 +28,7 @@ export default function RootLayout({ children }: { children: ReactNode }) { {/* biome-ignore lint/security/noDangerouslySetInnerHtml: Inline before hydration so the theme is correct on first paint. */}