diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md new file mode 100644 index 0000000..2091cfa --- /dev/null +++ b/.github/copilot-instructions.md @@ -0,0 +1,29 @@ +# Copilot Code Review Instructions + +## Security — PII and Secrets + +Flag any logging statements (`log::info!`, `log::debug!`, `log::warn!`, `log::error!`, +`tracing::info!`, `tracing::debug!`, `tracing::warn!`, `tracing::error!`, or unqualified +`info!`, `debug!`, `warn!`, `error!` macros (e.g., via `use tracing::{info, debug, warn, error}`)) +that may log: +- HTTP request/response headers (Authorization, Cookie, X-API-Key, or similar) +- HTTP request/response bodies or raw payloads +- Any PII fields (e.g., email, name, user_id, ip_address, phone, ssn, date_of_birth) +- API keys, tokens, secrets, or credentials +- Structs or types that contain any of the above fields +- `SendData` values or any variable that contains a `SendData` object (e.g., + `traces_with_tags` or similar variables built via `.with_api_key(...).build()`), + since these embed the Datadog API key + +Suggest redacting or omitting the sensitive field rather than logging it. + +## Security — Unsafe Rust + +Flag new `unsafe` blocks and explain what invariant the author must uphold to make the +block safe. If there is a safe alternative, suggest it. + +## Security — Error Handling + +Flag cases where errors are silently swallowed (empty `catch`, `.ok()` without +handling, `let _ = result`) or where operations like `.unwrap()`/`.expect()` may panic, +in code paths that handle external input or network responses. diff --git a/.github/workflows/build-datadog-serverless-compat.yml b/.github/workflows/build-datadog-serverless-compat.yml index 0c72908..8c88500 100644 --- a/.github/workflows/build-datadog-serverless-compat.yml +++ b/.github/workflows/build-datadog-serverless-compat.yml @@ -63,3 +63,25 @@ jobs: name: windows-amd64 path: target/release/datadog-serverless-compat.exe retention-days: 3 + - if: ${{ inputs.runner == 'windows-2022' }} + shell: bash + run: | + rustup target add i686-pc-windows-msvc + cargo build --release -p datadog-serverless-compat \ + --target i686-pc-windows-msvc \ + --features windows-pipes + - if: ${{ inputs.runner == 'windows-2022' }} + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # 4.6.2 + with: + name: windows-ia32 + path: target/i686-pc-windows-msvc/release/datadog-serverless-compat.exe + retention-days: 3 + - if: ${{ inputs.runner == 'macos-14' }} + shell: bash + run: cargo build --release -p datadog-serverless-compat + - if: ${{ inputs.runner == 'macos-14' }} + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # 4.6.2 + with: + name: darwin-arm64 + path: target/release/datadog-serverless-compat + retention-days: 3 diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 1d37e4d..4bcb69a 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -24,7 +24,7 @@ jobs: strategy: fail-fast: false matrix: - runner: [ubuntu-24.04, ubuntu-24.04-arm, windows-2022] + runner: [ubuntu-24.04, ubuntu-24.04-arm, windows-2022, macos-14] uses: ./.github/workflows/build-datadog-serverless-compat.yml with: runner: ${{ matrix.runner }} @@ -56,6 +56,16 @@ jobs: name: windows-amd64 path: target/windows-amd64 - run: upx target/windows-amd64/datadog-serverless-compat.exe --lzma + - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # 4.3.0 + with: + name: windows-ia32 + path: target/windows-ia32 + - run: upx target/windows-ia32/datadog-serverless-compat.exe --lzma + - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # 4.3.0 + with: + name: darwin-arm64 + path: target/darwin-arm64 + - run: chmod +x target/darwin-arm64/datadog-serverless-compat - name: Determine version id: determine-version env: @@ -85,6 +95,14 @@ jobs: mkdir -p npm/datadog-serverless-compat-win32-x64/bin cp target/windows-amd64/datadog-serverless-compat.exe npm/datadog-serverless-compat-win32-x64/bin/ npm --prefix npm/datadog-serverless-compat-win32-x64 pkg set version="$VERSION" + + mkdir -p npm/datadog-serverless-compat-win32-ia32/bin + cp target/windows-ia32/datadog-serverless-compat.exe npm/datadog-serverless-compat-win32-ia32/bin/ + npm --prefix npm/datadog-serverless-compat-win32-ia32 pkg set version="$VERSION" + + mkdir -p npm/datadog-serverless-compat-darwin-arm64/bin + cp target/darwin-arm64/datadog-serverless-compat npm/datadog-serverless-compat-darwin-arm64/bin/ + npm --prefix npm/datadog-serverless-compat-darwin-arm64 pkg set version="$VERSION" - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # 4.6.2 with: name: npm-packages @@ -110,3 +128,5 @@ jobs: npm publish ./npm/datadog-serverless-compat-linux-x64 --provenance --access public npm publish ./npm/datadog-serverless-compat-linux-arm64 --provenance --access public npm publish ./npm/datadog-serverless-compat-win32-x64 --provenance --access public + npm publish ./npm/datadog-serverless-compat-win32-ia32 --provenance --access public + npm publish ./npm/datadog-serverless-compat-darwin-arm64 --provenance --access public diff --git a/.gitignore b/.gitignore index df769d2..15ed4dd 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,6 @@ /target /.idea +/.worktrees +/CLAUDE.md +/AGENTS.md diff --git a/Cargo.lock b/Cargo.lock index 8a6e243..2df7122 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -89,7 +89,7 @@ checksum = "9035ad2d096bed7955a320ee7e2230574d28fd3c3a0f186cbea1ff3c7eed5dbb" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn", ] [[package]] @@ -179,7 +179,7 @@ dependencies = [ "regex", "rustc-hash", "shlex", - "syn 2.0.117", + "syn", ] [[package]] @@ -212,6 +212,12 @@ dependencies = [ "generic-array", ] +[[package]] +name = "borrow-or-share" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc0b364ead1874514c8c2855ab558056ebfeb775653e7ae45ff72f28f8f3166c" + [[package]] name = "bumpalo" version = "3.20.2" @@ -442,7 +448,7 @@ dependencies = [ "dogstatsd", "figment", "libdd-trace-obfuscation", - "libdd-trace-utils 1.0.0", + "libdd-trace-utils 3.0.1", "log", "serde", "serde-aux", @@ -461,6 +467,26 @@ dependencies = [ "tracing", ] +[[package]] +name = "datadog-logs-agent" +version = "0.1.0" +dependencies = [ + "datadog-fips", + "futures", + "http", + "http-body-util", + "hyper", + "hyper-util", + "mockito", + "reqwest", + "serde", + "serde_json", + "thiserror 1.0.69", + "tokio", + "tracing", + "zstd", +] + [[package]] name = "datadog-opentelemetry" version = "0.3.0" @@ -483,7 +509,7 @@ dependencies = [ "opentelemetry", "opentelemetry-semantic-conventions", "opentelemetry_sdk", - "rand 0.8.5", + "rand 0.8.6", "rustc_version_runtime", "serde", "serde_json", @@ -513,10 +539,12 @@ name = "datadog-serverless-compat" version = "0.1.0" dependencies = [ "datadog-fips", + "datadog-logs-agent", "datadog-trace-agent", "dogstatsd", - "libdd-trace-utils 1.0.0", + "libdd-trace-utils 3.0.1", "reqwest", + "serde_json", "tokio", "tokio-util", "tracing", @@ -537,10 +565,11 @@ dependencies = [ "hyper", "hyper-http-proxy", "hyper-util", - "libdd-common 1.1.0", + "libdd-capabilities", + "libdd-common 3.0.2", "libdd-trace-obfuscation", - "libdd-trace-protobuf 1.0.0", - "libdd-trace-utils 1.0.0", + "libdd-trace-protobuf 3.0.1", + "libdd-trace-utils 3.0.1", "reqwest", "rmp-serde", "serde", @@ -580,7 +609,7 @@ checksum = "cb7330aeadfbe296029522e6c40f315320aba36fc43a5b3632f3795348f3bd22" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn", "unicode-xid", ] @@ -602,7 +631,7 @@ checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn", ] [[package]] @@ -640,12 +669,13 @@ checksum = "92773504d58c093f6de2459af4af33faa518c13451eb8f2b5698ed3d36e7c813" [[package]] name = "duplicate" -version = "0.4.1" +version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a0a4be4cd710e92098de6ad258e6e7c24af11c29c5142f3c6f2a545652480ff8" +checksum = "8e92f10a49176cbffacaedabfaa11d51db1ea0f80a83c26e1873b43cd1742c24" dependencies = [ - "heck 0.4.1", - "proc-macro-error", + "heck", + "proc-macro2", + "proc-macro2-diagnostics", ] [[package]] @@ -744,6 +774,17 @@ dependencies = [ "num-traits", ] +[[package]] +name = "fluent-uri" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc74ac4d8359ae70623506d512209619e5cf8f347124910440dbc221714b328e" +dependencies = [ + "borrow-or-share", + "ref-cast", + "serde", +] + [[package]] name = "fnv" version = "1.0.7" @@ -833,7 +874,7 @@ checksum = "e835b70203e41293343137df5c0664546da5745f82ec9b84d40be8336958447b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn", ] [[package]] @@ -998,12 +1039,6 @@ dependencies = [ "http", ] -[[package]] -name = "heck" -version = "0.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" - [[package]] name = "heck" version = "0.5.0" @@ -1392,12 +1427,36 @@ version = "0.2.183" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b5b646652bf6661599e1da8901b3b9522896f01e736bad5f723fe7a3a27f899d" +[[package]] +name = "libdd-capabilities" +version = "0.1.0" +source = "git+https://github.com/DataDog/libdatadog?rev=986aab55cb7941d8453dffb59d35a70599d08665#986aab55cb7941d8453dffb59d35a70599d08665" +dependencies = [ + "anyhow", + "bytes", + "http", + "thiserror 1.0.69", +] + +[[package]] +name = "libdd-capabilities-impl" +version = "0.1.0" +source = "git+https://github.com/DataDog/libdatadog?rev=986aab55cb7941d8453dffb59d35a70599d08665#986aab55cb7941d8453dffb59d35a70599d08665" +dependencies = [ + "bytes", + "http", + "libdd-capabilities", + "libdd-common 3.0.2", +] + [[package]] name = "libdd-common" -version = "1.1.0" -source = "git+https://github.com/DataDog/libdatadog?rev=d52ee90209cb12a28bdda0114535c1a985a29d95#d52ee90209cb12a28bdda0114535c1a985a29d95" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e5593b91f61eee38cddc9fdcbc99c9fad697b5d925e226bd500d86b4295380b" dependencies = [ "anyhow", + "bytes", "cc", "const_format", "futures", @@ -1408,28 +1467,23 @@ dependencies = [ "http-body", "http-body-util", "hyper", - "hyper-rustls", "hyper-util", "libc", "nix", "pin-project", "regex", - "rustls", - "rustls-native-certs", "serde", "static_assertions", "thiserror 1.0.69", "tokio", - "tokio-rustls", "tower-service", "windows-sys 0.52.0", ] [[package]] name = "libdd-common" -version = "2.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e5593b91f61eee38cddc9fdcbc99c9fad697b5d925e226bd500d86b4295380b" +version = "3.0.2" +source = "git+https://github.com/DataDog/libdatadog?rev=986aab55cb7941d8453dffb59d35a70599d08665#986aab55cb7941d8453dffb59d35a70599d08665" dependencies = [ "anyhow", "bytes", @@ -1443,15 +1497,20 @@ dependencies = [ "http-body", "http-body-util", "hyper", + "hyper-rustls", "hyper-util", "libc", + "libdd-capabilities", "nix", "pin-project", "regex", + "rustls", + "rustls-native-certs", "serde", "static_assertions", "thiserror 1.0.69", "tokio", + "tokio-rustls", "tower-service", "windows-sys 0.52.0", ] @@ -1546,51 +1605,52 @@ dependencies = [ [[package]] name = "libdd-tinybytes" version = "1.1.0" -source = "git+https://github.com/DataDog/libdatadog?rev=d52ee90209cb12a28bdda0114535c1a985a29d95#d52ee90209cb12a28bdda0114535c1a985a29d95" +source = "git+https://github.com/DataDog/libdatadog?rev=986aab55cb7941d8453dffb59d35a70599d08665#986aab55cb7941d8453dffb59d35a70599d08665" dependencies = [ "serde", ] [[package]] name = "libdd-trace-normalization" -version = "1.0.0" -source = "git+https://github.com/DataDog/libdatadog?rev=d52ee90209cb12a28bdda0114535c1a985a29d95#d52ee90209cb12a28bdda0114535c1a985a29d95" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a737b43f01d6a0cbd1399c5b89863a5d2663fe7b19bf1d3ea28048abab396353" dependencies = [ "anyhow", - "libdd-trace-protobuf 1.0.0", + "libdd-trace-protobuf 2.0.0", ] [[package]] name = "libdd-trace-normalization" -version = "1.0.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a737b43f01d6a0cbd1399c5b89863a5d2663fe7b19bf1d3ea28048abab396353" +version = "2.0.0" +source = "git+https://github.com/DataDog/libdatadog?rev=986aab55cb7941d8453dffb59d35a70599d08665#986aab55cb7941d8453dffb59d35a70599d08665" dependencies = [ "anyhow", - "libdd-trace-protobuf 2.0.0", + "libdd-trace-protobuf 3.0.1", ] [[package]] name = "libdd-trace-obfuscation" -version = "1.0.0" -source = "git+https://github.com/DataDog/libdatadog?rev=d52ee90209cb12a28bdda0114535c1a985a29d95#d52ee90209cb12a28bdda0114535c1a985a29d95" +version = "2.0.0" +source = "git+https://github.com/DataDog/libdatadog?rev=986aab55cb7941d8453dffb59d35a70599d08665#986aab55cb7941d8453dffb59d35a70599d08665" dependencies = [ "anyhow", - "libdd-common 1.1.0", - "libdd-trace-protobuf 1.0.0", - "libdd-trace-utils 1.0.0", + "fluent-uri", + "libdd-common 3.0.2", + "libdd-trace-protobuf 3.0.1", + "libdd-trace-utils 3.0.1", "log", "percent-encoding", "regex", "serde", "serde_json", - "url", ] [[package]] name = "libdd-trace-protobuf" -version = "1.0.0" -source = "git+https://github.com/DataDog/libdatadog?rev=d52ee90209cb12a28bdda0114535c1a985a29d95#d52ee90209cb12a28bdda0114535c1a985a29d95" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d0a54921e03174f3ff7ad8506ff9e13637e546ef0b1f369ae463eacebda8e88" dependencies = [ "prost 0.14.3", "serde", @@ -1599,9 +1659,8 @@ dependencies = [ [[package]] name = "libdd-trace-protobuf" -version = "2.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5d0a54921e03174f3ff7ad8506ff9e13637e546ef0b1f369ae463eacebda8e88" +version = "3.0.1" +source = "git+https://github.com/DataDog/libdatadog?rev=986aab55cb7941d8453dffb59d35a70599d08665#986aab55cb7941d8453dffb59d35a70599d08665" dependencies = [ "prost 0.14.3", "serde", @@ -1622,26 +1681,23 @@ dependencies = [ [[package]] name = "libdd-trace-utils" -version = "1.0.0" -source = "git+https://github.com/DataDog/libdatadog?rev=d52ee90209cb12a28bdda0114535c1a985a29d95#d52ee90209cb12a28bdda0114535c1a985a29d95" +version = "2.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a59e9a0a41bb17d06fb85a70db3be04e53ddfb8f61a593939bb9677729214db" dependencies = [ "anyhow", "bytes", - "cargo-platform", - "cargo_metadata", - "flate2", "futures", "http", + "http-body", "http-body-util", - "httpmock", - "hyper", "indexmap", - "libdd-common 1.1.0", - "libdd-tinybytes 1.1.0 (git+https://github.com/DataDog/libdatadog?rev=d52ee90209cb12a28bdda0114535c1a985a29d95)", - "libdd-trace-normalization 1.0.0", - "libdd-trace-protobuf 1.0.0", + "libdd-common 2.0.1", + "libdd-tinybytes 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "libdd-trace-normalization 1.0.2", + "libdd-trace-protobuf 2.0.0", "prost 0.14.3", - "rand 0.8.5", + "rand 0.8.6", "rmp", "rmp-serde", "rmpv", @@ -1649,29 +1705,35 @@ dependencies = [ "serde_json", "tokio", "tracing", - "urlencoding", - "zstd", ] [[package]] name = "libdd-trace-utils" -version = "2.0.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a59e9a0a41bb17d06fb85a70db3be04e53ddfb8f61a593939bb9677729214db" +version = "3.0.1" +source = "git+https://github.com/DataDog/libdatadog?rev=986aab55cb7941d8453dffb59d35a70599d08665#986aab55cb7941d8453dffb59d35a70599d08665" dependencies = [ "anyhow", + "base64 0.22.1", "bytes", + "cargo-platform", + "cargo_metadata", + "flate2", "futures", + "getrandom 0.2.17", "http", "http-body", "http-body-util", + "httpmock", + "hyper", "indexmap", - "libdd-common 2.0.1", - "libdd-tinybytes 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "libdd-trace-normalization 1.0.2", - "libdd-trace-protobuf 2.0.0", + "libdd-capabilities", + "libdd-capabilities-impl", + "libdd-common 3.0.2", + "libdd-tinybytes 1.1.0 (git+https://github.com/DataDog/libdatadog?rev=986aab55cb7941d8453dffb59d35a70599d08665)", + "libdd-trace-normalization 2.0.0", + "libdd-trace-protobuf 3.0.1", "prost 0.14.3", - "rand 0.8.5", + "rand 0.8.6", "rmp", "rmp-serde", "rmpv", @@ -1679,6 +1741,8 @@ dependencies = [ "serde_json", "tokio", "tracing", + "urlencoding", + "zstd", ] [[package]] @@ -1803,7 +1867,7 @@ dependencies = [ "hyper-util", "log", "pin-project-lite", - "rand 0.9.2", + "rand 0.9.4", "regex", "serde_json", "serde_urlencoded", @@ -1865,9 +1929,9 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "openssl-probe" -version = "0.2.1" +version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7c87def4c32ab89d880effc9e097653c8da5d6ef28e6b539d313baaacfbafcbe" +checksum = "d05e27ee213611ffe7d6348b942e8f942b37114c00cc03cec254295a4a17852e" [[package]] name = "opentelemetry" @@ -1899,7 +1963,7 @@ dependencies = [ "futures-util", "opentelemetry", "percent-encoding", - "rand 0.9.2", + "rand 0.9.4", "thiserror 2.0.18", ] @@ -1979,7 +2043,7 @@ dependencies = [ "proc-macro2", "proc-macro2-diagnostics", "quote", - "syn 2.0.117", + "syn", ] [[package]] @@ -2015,7 +2079,7 @@ checksum = "d9b20ed30f105399776b9c883e68e536ef602a16ae6f596d2c473591d6ad64c6" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn", ] [[package]] @@ -2061,31 +2125,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" dependencies = [ "proc-macro2", - "syn 2.0.117", -] - -[[package]] -name = "proc-macro-error" -version = "1.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da25490ff9892aab3fcf7c36f08cfb902dd3e71ca0f9f9517bea02a73a5ce38c" -dependencies = [ - "proc-macro-error-attr", - "proc-macro2", - "quote", - "syn 1.0.109", - "version_check", -] - -[[package]] -name = "proc-macro-error-attr" -version = "1.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a1be40180e52ecc98ad80b184934baf3d0d29f979574e439af5a55274b35f869" -dependencies = [ - "proc-macro2", - "quote", - "version_check", + "syn", ] [[package]] @@ -2105,7 +2145,7 @@ checksum = "af066a9c399a26e020ada66a034357a868728e72cd426f3adcd35f80d88d88c8" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn", "version_check", "yansi", ] @@ -2120,7 +2160,7 @@ dependencies = [ "bit-vec", "bitflags", "num-traits", - "rand 0.9.2", + "rand 0.9.4", "rand_chacha 0.9.0", "rand_xorshift", "regex-syntax", @@ -2155,7 +2195,7 @@ version = "0.13.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "be769465445e8c1474e9c5dac2018218498557af32d9ed057325ec9a41ae81bf" dependencies = [ - "heck 0.5.0", + "heck", "itertools 0.14.0", "log", "multimap", @@ -2165,7 +2205,7 @@ dependencies = [ "prost 0.13.5", "prost-types", "regex", - "syn 2.0.117", + "syn", "tempfile", ] @@ -2179,7 +2219,7 @@ dependencies = [ "itertools 0.14.0", "proc-macro2", "quote", - "syn 2.0.117", + "syn", ] [[package]] @@ -2192,7 +2232,7 @@ dependencies = [ "itertools 0.14.0", "proc-macro2", "quote", - "syn 2.0.117", + "syn", ] [[package]] @@ -2291,7 +2331,7 @@ dependencies = [ "bytes", "getrandom 0.3.4", "lru-slab", - "rand 0.9.2", + "rand 0.9.4", "ring", "rustc-hash", "rustls", @@ -2340,9 +2380,9 @@ checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" [[package]] name = "rand" -version = "0.8.5" +version = "0.8.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" +checksum = "5ca0ecfa931c29007047d1bc58e623ab12e5590e8c7cc53200d5202b69266d8a" dependencies = [ "libc", "rand_chacha 0.3.1", @@ -2351,9 +2391,9 @@ dependencies = [ [[package]] name = "rand" -version = "0.9.2" +version = "0.9.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6db2770f06117d490610c7488547d543617b21bfa07796d7a12f6f1bd53850d1" +checksum = "44c5af06bb1b7d3216d91932aed5265164bf384dc89cd6ba05cf59a35f5f76ea" dependencies = [ "rand_chacha 0.9.0", "rand_core 0.9.5", @@ -2415,6 +2455,26 @@ dependencies = [ "bitflags", ] +[[package]] +name = "ref-cast" +version = "1.0.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f354300ae66f76f1c85c5f84693f0ce81d747e2c3f21a45fef496d89c960bf7d" +dependencies = [ + "ref-cast-impl", +] + +[[package]] +name = "ref-cast-impl" +version = "1.0.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7186006dcb21920990093f30e3dea63b7d6e977bf1256be20c3563a5db070da" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "regex" version = "1.12.3" @@ -2593,9 +2653,9 @@ dependencies = [ [[package]] name = "rustls-native-certs" -version = "0.8.3" +version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "612460d5f7bea540c490b2b6395d8e34a953e52b491accd6c86c8164c5932a63" +checksum = "9980d917ebb0c0536119ba501e90834767bffc3d60641457fd84a1f3fd337923" dependencies = [ "openssl-probe", "rustls-pki-types", @@ -2615,9 +2675,9 @@ dependencies = [ [[package]] name = "rustls-webpki" -version = "0.103.10" +version = "0.103.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df33b2b81ac578cabaf06b89b0631153a3f416b0a886e8a7a1707fb51abbd1ef" +checksum = "8279bb85272c9f10811ae6a6c547ff594d6a7f3c6c6b02ee9726d1d0dcfcdd06" dependencies = [ "aws-lc-rs", "ring", @@ -2755,7 +2815,7 @@ checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn", ] [[package]] @@ -2764,6 +2824,7 @@ version = "1.0.149" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "83fc039473c5595ace860d8c4fafa220ff474b3fc6bfdb4293327f1a37e94d86" dependencies = [ + "indexmap", "itoa", "memchr", "serde", @@ -2828,7 +2889,7 @@ checksum = "91d129178576168c589c9ec973feedf7d3126c01ac2bf08795109aa35b69fb8f" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn", ] [[package]] @@ -2936,16 +2997,6 @@ version = "2.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" -[[package]] -name = "syn" -version = "1.0.109" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" -dependencies = [ - "proc-macro2", - "unicode-ident", -] - [[package]] name = "syn" version = "2.0.117" @@ -2974,7 +3025,7 @@ checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn", ] [[package]] @@ -3044,7 +3095,7 @@ checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn", ] [[package]] @@ -3055,7 +3106,7 @@ checksum = "ebc4ee7f67670e9b64d05fa4253e753e016c6c95ff35b89b7941d6b856dec1d5" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn", ] [[package]] @@ -3116,7 +3167,7 @@ checksum = "5c55a2eff8b69ce66c84f85e1da1c233edc36ceb85a2058d11b0d6a3c7e7569c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn", ] [[package]] @@ -3185,7 +3236,7 @@ dependencies = [ "prost-build", "prost-types", "quote", - "syn 2.0.117", + "syn", ] [[package]] @@ -3253,7 +3304,7 @@ checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn", ] [[package]] @@ -3313,7 +3364,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ad06847b7afb65c7866a36664b75c40b895e318cea4f71299f013fb22965329d" dependencies = [ "quote", - "syn 2.0.117", + "syn", ] [[package]] @@ -3520,7 +3571,7 @@ dependencies = [ "bumpalo", "proc-macro2", "quote", - "syn 2.0.117", + "syn", "wasm-bindgen-shared", ] @@ -3879,7 +3930,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ea61de684c3ea68cb082b7a88508a8b27fcc8b797d738bfc99a82facf1d752dc" dependencies = [ "anyhow", - "heck 0.5.0", + "heck", "wit-parser", ] @@ -3890,10 +3941,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b7c566e0f4b284dd6561c786d9cb0142da491f46a9fbed79ea69cdad5db17f21" dependencies = [ "anyhow", - "heck 0.5.0", + "heck", "indexmap", "prettyplease", - "syn 2.0.117", + "syn", "wasm-metadata", "wit-bindgen-core", "wit-component", @@ -3909,7 +3960,7 @@ dependencies = [ "prettyplease", "proc-macro2", "quote", - "syn 2.0.117", + "syn", "wit-bindgen-core", "wit-bindgen-rust", ] @@ -3982,7 +4033,7 @@ checksum = "b659052874eb698efe5b9e8cf382204678a0086ebf46982b79d6ca3182927e5d" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn", "synstructure", ] @@ -4003,7 +4054,7 @@ checksum = "0e8bc7269b54418e7aeeef514aa68f8690b8c0489a06b0136e5f57c4c5ccab89" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn", ] [[package]] @@ -4023,7 +4074,7 @@ checksum = "d71e5d6e06ab090c67b5e44993ec16b72dcbaabc526db883a360057678b48502" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn", "synstructure", ] @@ -4063,7 +4114,7 @@ checksum = "eadce39539ca5cb3985590102671f2567e659fca9666581ad3411d59207951f3" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn", ] [[package]] diff --git a/LICENSE-3rdparty.csv b/LICENSE-3rdparty.csv index 012085e..c89c5e2 100644 --- a/LICENSE-3rdparty.csv +++ b/LICENSE-3rdparty.csv @@ -20,6 +20,7 @@ bit-set,https://github.com/contain-rs/bit-set,Apache-2.0 OR MIT,Alexis Beingessn bit-vec,https://github.com/contain-rs/bit-vec,Apache-2.0 OR MIT,Alexis Beingessner bitflags,https://github.com/bitflags/bitflags,MIT OR Apache-2.0,The Rust Project Developers block-buffer,https://github.com/RustCrypto/utils,MIT OR Apache-2.0,RustCrypto Developers +borrow-or-share,https://github.com/yescallop/borrow-or-share,MIT-0,Scallop Ye bumpalo,https://github.com/fitzgen/bumpalo,MIT OR Apache-2.0,Nick Fitzgerald bytemuck,https://github.com/Lokathor/bytemuck,Zlib OR Apache-2.0 OR MIT,Lokathor byteorder,https://github.com/BurntSushi/byteorder,Unlicense OR MIT,Andrew Gallant @@ -61,6 +62,7 @@ find-msvc-tools,https://github.com/rust-lang/cc-rs,MIT OR Apache-2.0,The find-ms fixedbitset,https://github.com/petgraph/fixedbitset,MIT OR Apache-2.0,bluss flate2,https://github.com/rust-lang/flate2-rs,MIT OR Apache-2.0,"Alex Crichton , Josh Triplett " float-cmp,https://github.com/mikedilger/float-cmp,MIT,Mike Dilger +fluent-uri,https://github.com/yescallop/fluent-uri-rs,MIT,Scallop Ye fnv,https://github.com/servo/rust-fnv,Apache-2.0 OR MIT,Alex Crichton foldhash,https://github.com/orlp/foldhash,Zlib,Orson Peters form_urlencoded,https://github.com/servo/rust-url,MIT OR Apache-2.0,The rust-url developers @@ -82,7 +84,6 @@ hashbrown,https://github.com/rust-lang/hashbrown,MIT OR Apache-2.0,Amanieu d'Ant headers,https://github.com/hyperium/headers,MIT,Sean McArthur headers-core,https://github.com/hyperium/headers,MIT,Sean McArthur heck,https://github.com/withoutboats/heck,MIT OR Apache-2.0,The heck Authors -heck,https://github.com/withoutboats/heck,MIT OR Apache-2.0,Without Boats hex,https://github.com/KokaKiwi/rust-hex,MIT OR Apache-2.0,KokaKiwi home,https://github.com/rust-lang/cargo,MIT OR Apache-2.0,Brian Anderson http,https://github.com/hyperium/http,MIT OR Apache-2.0,"Alex Crichton , Carl Lerche , Sean McArthur " @@ -116,6 +117,8 @@ js-sys,https://github.com/wasm-bindgen/wasm-bindgen/tree/master/crates/js-sys,MI lazy_static,https://github.com/rust-lang-nursery/lazy-static.rs,MIT OR Apache-2.0,Marvin Löbel leb128fmt,https://github.com/bluk/leb128fmt,MIT OR Apache-2.0,Bryant Luk libc,https://github.com/rust-lang/libc,MIT OR Apache-2.0,The Rust Project Developers +libdd-capabilities,https://github.com/DataDog/libdatadog/tree/main/libdd-capabilities,Apache-2.0,The libdd-capabilities Authors +libdd-capabilities-impl,https://github.com/DataDog/libdatadog/tree/main/libdd-capabilities-impl,Apache-2.0,The libdd-capabilities-impl Authors libdd-common,https://github.com/DataDog/libdatadog/tree/main/datadog-common,Apache-2.0,The libdd-common Authors libdd-data-pipeline,https://github.com/DataDog/libdatadog/tree/main/libdd-data-pipeline,Apache-2.0,The libdd-data-pipeline Authors libdd-ddsketch,https://github.com/DataDog/libdatadog/tree/main/libdd-ddsketch,Apache-2.0,The libdd-ddsketch Authors @@ -146,7 +149,7 @@ nom,https://github.com/Geal/nom,MIT,contact@geoffroycouprie.com nu-ansi-term,https://github.com/nushell/nu-ansi-term,MIT,"ogham@bsago.me, Ryan Scheel (Havvy) , Josh Triplett , The Nushell Project Developers" num-traits,https://github.com/rust-num/num-traits,MIT OR Apache-2.0,The Rust Project Developers once_cell,https://github.com/matklad/once_cell,MIT OR Apache-2.0,Aleksey Kladov -openssl-probe,https://github.com/rustls/openssl-probe,MIT OR Apache-2.0,Alex Crichton +openssl-probe,https://github.com/alexcrichton/openssl-probe,MIT OR Apache-2.0,Alex Crichton opentelemetry,https://github.com/open-telemetry/opentelemetry-rust/tree/main/opentelemetry,Apache-2.0,The opentelemetry Authors opentelemetry-semantic-conventions,https://github.com/open-telemetry/opentelemetry-rust/tree/main/opentelemetry-semantic-conventions,Apache-2.0,The opentelemetry-semantic-conventions Authors opentelemetry_sdk,https://github.com/open-telemetry/opentelemetry-rust/tree/main/opentelemetry-sdk,Apache-2.0,The opentelemetry_sdk Authors @@ -166,8 +169,6 @@ pin-utils,https://github.com/rust-lang-nursery/pin-utils,MIT OR Apache-2.0,Josef potential_utf,https://github.com/unicode-org/icu4x,Unicode-3.0,The ICU4X Project Developers ppv-lite86,https://github.com/cryptocorrosion/cryptocorrosion,MIT OR Apache-2.0,The CryptoCorrosion Contributors prettyplease,https://github.com/dtolnay/prettyplease,MIT OR Apache-2.0,David Tolnay -proc-macro-error,https://gitlab.com/CreepySkeleton/proc-macro-error,MIT OR Apache-2.0,CreepySkeleton -proc-macro-error-attr,https://gitlab.com/CreepySkeleton/proc-macro-error,MIT OR Apache-2.0,CreepySkeleton proc-macro2,https://github.com/dtolnay/proc-macro2,MIT OR Apache-2.0,"David Tolnay , Alex Crichton " proc-macro2-diagnostics,https://github.com/SergioBenitez/proc-macro2-diagnostics,MIT OR Apache-2.0,Sergio Benitez prost,https://github.com/tokio-rs/prost,Apache-2.0,"Dan Burkert , Lucio Franco , Casper Meijn , Tokio Contributors " @@ -188,6 +189,8 @@ rand_chacha,https://github.com/rust-random/rand,MIT OR Apache-2.0,"The Rand Proj rand_core,https://github.com/rust-random/rand,MIT OR Apache-2.0,"The Rand Project Developers, The Rust Project Developers" rand_xorshift,https://github.com/rust-random/rngs,MIT OR Apache-2.0,"The Rand Project Developers, The Rust Project Developers" redox_syscall,https://gitlab.redox-os.org/redox-os/syscall,MIT,Jeremy Soller +ref-cast,https://github.com/dtolnay/ref-cast,MIT OR Apache-2.0,David Tolnay +ref-cast-impl,https://github.com/dtolnay/ref-cast,MIT OR Apache-2.0,David Tolnay regex,https://github.com/rust-lang/regex,MIT OR Apache-2.0,"The Rust Project Developers, Andrew Gallant " regex-automata,https://github.com/rust-lang/regex,MIT OR Apache-2.0,"The Rust Project Developers, Andrew Gallant " regex-syntax,https://github.com/rust-lang/regex,MIT OR Apache-2.0,"The Rust Project Developers, Andrew Gallant " diff --git a/crates/datadog-agent-config/Cargo.toml b/crates/datadog-agent-config/Cargo.toml index 089537b..bd87bcb 100644 --- a/crates/datadog-agent-config/Cargo.toml +++ b/crates/datadog-agent-config/Cargo.toml @@ -4,13 +4,10 @@ version = "0.1.0" edition.workspace = true license.workspace = true -[lib] -path = "mod.rs" - [dependencies] figment = { version = "0.10", default-features = false, features = ["yaml", "env"] } -libdd-trace-obfuscation = { git = "https://github.com/DataDog/libdatadog", rev = "d52ee90209cb12a28bdda0114535c1a985a29d95" } -libdd-trace-utils = { git = "https://github.com/DataDog/libdatadog", rev = "d52ee90209cb12a28bdda0114535c1a985a29d95" } +libdd-trace-obfuscation = { git = "https://github.com/DataDog/libdatadog", rev = "986aab55cb7941d8453dffb59d35a70599d08665" } +libdd-trace-utils = { git = "https://github.com/DataDog/libdatadog", rev = "986aab55cb7941d8453dffb59d35a70599d08665" } log = { version = "0.4", default-features = false } serde = { version = "1.0", default-features = false, features = ["derive"] } serde-aux = { version = "4.7", default-features = false } diff --git a/crates/datadog-agent-config/README.md b/crates/datadog-agent-config/README.md new file mode 100644 index 0000000..3dd478e --- /dev/null +++ b/crates/datadog-agent-config/README.md @@ -0,0 +1,117 @@ +# datadog-agent-config + +Shared configuration crate for Datadog serverless agents. Provides a typed `Config` struct with built-in loading from environment variables (`DD_*`) and YAML files (`datadog.yaml`), with environment variables taking precedence. + +## Core features + +- **Typed config struct** with fields for site, API key, proxy, logs, APM, metrics, DogStatsD, OTLP, and trace propagation +- **Two built-in sources**: `EnvConfigSource` (reads `DD_*` / `DATADOG_*` env vars) and `YamlConfigSource` (reads `datadog.yaml`) +- **Graceful deserialization**: every field uses forgiving deserializers that fall back to defaults on bad input, so one misconfigured value never crashes the whole config +- **Extensible via `ConfigExtension`**: consumers can define additional configuration fields without modifying this crate + +## Quick start + +```rust +use std::path::Path; +use datadog_agent_config::get_config; + +let config = get_config(Path::new("/var/task")); +println!("site: {}", config.site); +println!("api_key: {}", config.api_key); +``` + +## Extensible configuration + +Consumers that need additional fields (e.g., Lambda-specific settings) implement the `ConfigExtension` trait instead of forking or copy-pasting the crate. + +### 1. Define the extension and its source + +```rust +use datadog_agent_config::{ + ConfigExtension, merge_fields, + deserialize_optional_string, deserialize_optional_bool_from_anything, +}; +use serde::Deserialize; + +#[derive(Debug, PartialEq, Clone)] +pub struct MyExtension { + pub custom_flag: bool, + pub custom_name: String, +} + +impl Default for MyExtension { + fn default() -> Self { + Self { custom_flag: false, custom_name: String::new() } + } +} + +/// Source struct for deserialization. +/// +/// REQUIRED: `#[serde(default)]` on the struct + graceful deserializers on each +/// field. Without these, a missing or malformed value fails the entire extension +/// extraction — fields silently fall back to defaults with a warning log. +#[derive(Debug, Clone, Default, Deserialize)] +#[serde(default)] +pub struct MySource { + #[serde(deserialize_with = "deserialize_optional_bool_from_anything")] + pub custom_flag: Option, + #[serde(deserialize_with = "deserialize_optional_string")] + pub custom_name: Option, +} + +impl ConfigExtension for MyExtension { + type Source = MySource; + + fn merge_from(&mut self, source: &MySource) { + merge_fields!(self, source, + string: [custom_name], + value: [custom_flag], + ); + } +} +``` + +### 2. Load config with the extension + +```rust +use std::path::Path; +use datadog_agent_config::{Config, get_config_with_extension}; + +type MyConfig = Config; + +let config: MyConfig = get_config_with_extension(Path::new("/var/task")); + +// Core fields +println!("site: {}", config.site); + +// Extension fields +println!("custom_flag: {}", config.ext.custom_flag); +println!("custom_name: {}", config.ext.custom_name); +``` + +Extension fields are populated from both `DD_*` environment variables and `datadog.yaml` using dual extraction: the core fields and extension fields are extracted independently from the same figment instance, so they don't interfere with each other. + +### Flat fields only + +The single `Source` type is used for both env var and YAML extraction. This works because Figment uses a single key-value namespace per provider, so flat fields map naturally to both `DD_*` env vars and top-level YAML keys. If you need nested YAML structures (e.g., `lambda: { enhanced_metrics: true }`) that differ from the flat env var layout, you'd need separate source structs — implement `merge_from` with a nested source struct and handle the mapping manually. + +### Field name collisions + +Extension fields are extracted independently from the same figment as core fields. If an extension defines a field with the same name as a core field (e.g., `api_key`), both get their own copy — they don't interfere, but the extension copy does **not** override the core value. Avoid shadowing core field names to prevent confusion. + +### merge_fields! macro + +The `merge_fields!` macro reduces boilerplate in `merge_from` by batching fields by merge strategy: + +- `string`: merges `Option` into `String` (sets value if `Some`) +- `value`: merges `Option` into `T` (sets value if `Some`) +- `option`: merges `Option` into `Option` (overwrites if `Some`) + +Custom merge logic (e.g., OR-ing two boolean fields together) goes after the macro call in the same method. + +## Config loading precedence + +1. `Config::default()` (hardcoded defaults) +2. `datadog.yaml` values (lower priority) +3. `DD_*` environment variables (highest priority) +4. Post-processing defaults (site, proxy, logs/APM URL construction) diff --git a/crates/datadog-agent-config/additional_endpoints.rs b/crates/datadog-agent-config/src/deserializers/additional_endpoints.rs similarity index 100% rename from crates/datadog-agent-config/additional_endpoints.rs rename to crates/datadog-agent-config/src/deserializers/additional_endpoints.rs diff --git a/crates/datadog-agent-config/apm_replace_rule.rs b/crates/datadog-agent-config/src/deserializers/apm_replace_rule.rs similarity index 100% rename from crates/datadog-agent-config/apm_replace_rule.rs rename to crates/datadog-agent-config/src/deserializers/apm_replace_rule.rs diff --git a/crates/datadog-agent-config/flush_strategy.rs b/crates/datadog-agent-config/src/deserializers/flush_strategy.rs similarity index 100% rename from crates/datadog-agent-config/flush_strategy.rs rename to crates/datadog-agent-config/src/deserializers/flush_strategy.rs diff --git a/crates/datadog-agent-config/src/deserializers/helpers.rs b/crates/datadog-agent-config/src/deserializers/helpers.rs new file mode 100644 index 0000000..058c0ce --- /dev/null +++ b/crates/datadog-agent-config/src/deserializers/helpers.rs @@ -0,0 +1,372 @@ +use serde::{Deserialize, Deserializer}; +use serde_aux::prelude::deserialize_bool_from_anything; +use serde_json::Value; + +use std::collections::HashMap; +use std::fmt; +use std::time::Duration; +use tracing::warn; + +use crate::TracePropagationStyle; + +pub fn deserialize_optional_string<'de, D>(deserializer: D) -> Result, D::Error> +where + D: Deserializer<'de>, +{ + match Value::deserialize(deserializer)? { + Value::String(s) => Ok(Some(s)), + other => { + warn!( + "Failed to parse value, expected a string, got: {}, ignoring", + other + ); + Ok(None) + } + } +} + +pub fn deserialize_string_or_int<'de, D>(deserializer: D) -> Result, D::Error> +where + D: Deserializer<'de>, +{ + let value = Value::deserialize(deserializer)?; + match value { + Value::String(s) => { + if s.trim().is_empty() { + Ok(None) + } else { + Ok(Some(s)) + } + } + Value::Number(n) => Ok(Some(n.to_string())), + _ => { + warn!("Failed to parse value, expected a string or an integer, ignoring"); + Ok(None) + } + } +} + +pub fn deserialize_optional_bool_from_anything<'de, D>( + deserializer: D, +) -> Result, D::Error> +where + D: Deserializer<'de>, +{ + // First try to deserialize as Option<_> to handle null/missing values + let opt: Option = Option::deserialize(deserializer)?; + + match opt { + None => Ok(None), + Some(value) => match deserialize_bool_from_anything(value) { + Ok(bool_result) => Ok(Some(bool_result)), + Err(e) => { + warn!("Failed to parse bool value: {}, ignoring", e); + Ok(None) + } + }, + } +} + +/// Parse a single "key:value" string into a (key, value) tuple +/// Returns None if the string is invalid (e.g., missing colon, empty key/value) +fn parse_key_value_tag(tag: &str) -> Option<(String, String)> { + let parts: Vec<&str> = tag.splitn(2, ':').collect(); + if parts.len() == 2 && !parts[0].is_empty() && !parts[1].is_empty() { + Some((parts[0].to_string(), parts[1].to_string())) + } else { + warn!( + "Failed to parse tag '{}', expected format 'key:value', ignoring", + tag + ); + None + } +} + +pub fn deserialize_key_value_pairs<'de, D>( + deserializer: D, +) -> Result, D::Error> +where + D: Deserializer<'de>, +{ + struct KeyValueVisitor; + + impl serde::de::Visitor<'_> for KeyValueVisitor { + type Value = HashMap; + + fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("a string in format 'key1:value1,key2:value2' or 'key1:value1'") + } + + fn visit_str(self, value: &str) -> Result + where + E: serde::de::Error, + { + let mut map = HashMap::new(); + for tag in value.split(&[',', ' ']) { + if tag.is_empty() { + continue; + } + if let Some((key, val)) = parse_key_value_tag(tag) { + map.insert(key, val); + } + } + + Ok(map) + } + + fn visit_u64(self, value: u64) -> Result + where + E: serde::de::Error, + { + warn!( + "Failed to parse tags: expected string in format 'key:value', got number {}, ignoring", + value + ); + Ok(HashMap::new()) + } + + fn visit_i64(self, value: i64) -> Result + where + E: serde::de::Error, + { + warn!( + "Failed to parse tags: expected string in format 'key:value', got number {}, ignoring", + value + ); + Ok(HashMap::new()) + } + + fn visit_f64(self, value: f64) -> Result + where + E: serde::de::Error, + { + warn!( + "Failed to parse tags: expected string in format 'key:value', got number {}, ignoring", + value + ); + Ok(HashMap::new()) + } + + fn visit_bool(self, value: bool) -> Result + where + E: serde::de::Error, + { + warn!( + "Failed to parse tags: expected string in format 'key:value', got boolean {}, ignoring", + value + ); + Ok(HashMap::new()) + } + } + + deserializer.deserialize_any(KeyValueVisitor) +} + +pub fn deserialize_array_from_comma_separated_string<'de, D>( + deserializer: D, +) -> Result, D::Error> +where + D: Deserializer<'de>, +{ + let s: String = String::deserialize(deserializer)?; + Ok(s.split(',') + .map(|feature| feature.trim().to_string()) + .filter(|feature| !feature.is_empty()) + .collect()) +} + +pub fn deserialize_key_value_pair_array_to_hashmap<'de, D>( + deserializer: D, +) -> Result, D::Error> +where + D: Deserializer<'de>, +{ + let array: Vec = match Vec::deserialize(deserializer) { + Ok(v) => v, + Err(e) => { + warn!("Failed to deserialize tags array: {e}, ignoring"); + return Ok(HashMap::new()); + } + }; + let mut map = HashMap::new(); + for s in array { + if let Some((key, val)) = parse_key_value_tag(&s) { + map.insert(key, val); + } + } + Ok(map) +} + +/// Deserialize APM filter tags from space-separated "key:value" pairs, also support key-only tags +pub fn deserialize_apm_filter_tags<'de, D>(deserializer: D) -> Result>, D::Error> +where + D: Deserializer<'de>, +{ + let opt: Option = Option::deserialize(deserializer)?; + + match opt { + None => Ok(None), + Some(s) if s.trim().is_empty() => Ok(None), + Some(s) => { + let tags: Vec = s + .split_whitespace() + .filter_map(|pair| { + let parts: Vec<&str> = pair.splitn(2, ':').collect(); + if parts.len() == 2 { + let key = parts[0].trim(); + let value = parts[1].trim(); + if key.is_empty() { + None + } else if value.is_empty() { + Some(key.to_string()) + } else { + Some(format!("{key}:{value}")) + } + } else if parts.len() == 1 { + let key = parts[0].trim(); + if key.is_empty() { + None + } else { + Some(key.to_string()) + } + } else { + None + } + }) + .collect(); + + if tags.is_empty() { + Ok(None) + } else { + Ok(Some(tags)) + } + } + } +} + +pub fn deserialize_option_lossless<'de, D, T>(deserializer: D) -> Result, D::Error> +where + D: Deserializer<'de>, + T: Deserialize<'de>, +{ + match Option::::deserialize(deserializer) { + Ok(value) => Ok(value), + Err(e) => { + warn!("Failed to deserialize optional value: {}, ignoring", e); + Ok(None) + } + } +} + +/// Gracefully deserialize any field, falling back to `T::default()` on error. +/// +/// This ensures that a single field with the wrong type never fails the entire +/// struct extraction. Works for any `T` that implements `Deserialize + Default`: +/// - `Option` defaults to `None` +/// - `Vec` defaults to `[]` +/// - `HashMap` defaults to `{}` +/// - Structs with `#[derive(Default)]` use their default +pub fn deserialize_with_default<'de, D, T>(deserializer: D) -> Result +where + D: Deserializer<'de>, + T: Deserialize<'de> + Default, +{ + match T::deserialize(deserializer) { + Ok(value) => Ok(value), + Err(e) => { + warn!("Failed to deserialize field: {}, using default", e); + Ok(T::default()) + } + } +} + +pub fn deserialize_optional_duration_from_microseconds<'de, D: Deserializer<'de>>( + deserializer: D, +) -> Result, D::Error> { + match Option::::deserialize(deserializer) { + Ok(opt) => Ok(opt.map(Duration::from_micros)), + Err(e) => { + warn!("Failed to deserialize duration (microseconds): {e}, ignoring"); + Ok(None) + } + } +} + +pub fn deserialize_optional_duration_from_seconds<'de, D: Deserializer<'de>>( + deserializer: D, +) -> Result, D::Error> { + // Deserialize into a generic Value first to avoid propagating type errors, + // then try to extract a duration from it. + match Value::deserialize(deserializer) { + Ok(Value::Number(n)) => { + if let Some(u) = n.as_u64() { + Ok(Some(Duration::from_secs(u))) + } else if let Some(i) = n.as_i64() { + if i < 0 { + warn!("Failed to parse duration: negative durations are not allowed, ignoring"); + Ok(None) + } else { + Ok(Some(Duration::from_secs(i as u64))) + } + } else if let Some(f) = n.as_f64() { + if f < 0.0 { + warn!("Failed to parse duration: negative durations are not allowed, ignoring"); + Ok(None) + } else { + Ok(Some(Duration::from_secs_f64(f))) + } + } else { + warn!("Failed to parse duration: unsupported number format, ignoring"); + Ok(None) + } + } + Ok(Value::Null) => Ok(None), + Ok(other) => { + warn!("Failed to parse duration: expected number, got {other}, ignoring"); + Ok(None) + } + Err(e) => { + warn!("Failed to deserialize duration: {e}, ignoring"); + Ok(None) + } + } +} + +// Like deserialize_optional_duration_from_seconds(), but return None if the value is 0 +pub fn deserialize_optional_duration_from_seconds_ignore_zero<'de, D: Deserializer<'de>>( + deserializer: D, +) -> Result, D::Error> { + let duration: Option = deserialize_optional_duration_from_seconds(deserializer)?; + if duration.is_some_and(|d| d.as_secs() == 0) { + return Ok(None); + } + Ok(duration) +} + +pub fn deserialize_trace_propagation_style<'de, D>( + deserializer: D, +) -> Result, D::Error> +where + D: Deserializer<'de>, +{ + use std::str::FromStr; + let s: String = match String::deserialize(deserializer) { + Ok(s) => s, + Err(e) => { + warn!("Failed to deserialize trace propagation style: {e}, ignoring"); + return Ok(Vec::new()); + } + }; + + Ok(s.split(',') + .filter_map( + |style| match TracePropagationStyle::from_str(style.trim()) { + Ok(parsed_style) => Some(parsed_style), + Err(e) => { + warn!("Failed to parse trace propagation style: {e}, ignoring"); + None + } + }, + ) + .collect()) +} diff --git a/crates/datadog-agent-config/log_level.rs b/crates/datadog-agent-config/src/deserializers/log_level.rs similarity index 100% rename from crates/datadog-agent-config/log_level.rs rename to crates/datadog-agent-config/src/deserializers/log_level.rs diff --git a/crates/datadog-agent-config/logs_additional_endpoints.rs b/crates/datadog-agent-config/src/deserializers/logs_additional_endpoints.rs similarity index 100% rename from crates/datadog-agent-config/logs_additional_endpoints.rs rename to crates/datadog-agent-config/src/deserializers/logs_additional_endpoints.rs diff --git a/crates/datadog-agent-config/src/deserializers/mod.rs b/crates/datadog-agent-config/src/deserializers/mod.rs new file mode 100644 index 0000000..e88c90b --- /dev/null +++ b/crates/datadog-agent-config/src/deserializers/mod.rs @@ -0,0 +1,8 @@ +pub mod additional_endpoints; +pub mod apm_replace_rule; +pub mod flush_strategy; +pub mod helpers; +pub mod log_level; +pub mod logs_additional_endpoints; +pub mod processing_rule; +pub mod service_mapping; diff --git a/crates/datadog-agent-config/processing_rule.rs b/crates/datadog-agent-config/src/deserializers/processing_rule.rs similarity index 100% rename from crates/datadog-agent-config/processing_rule.rs rename to crates/datadog-agent-config/src/deserializers/processing_rule.rs diff --git a/crates/datadog-agent-config/service_mapping.rs b/crates/datadog-agent-config/src/deserializers/service_mapping.rs similarity index 100% rename from crates/datadog-agent-config/service_mapping.rs rename to crates/datadog-agent-config/src/deserializers/service_mapping.rs diff --git a/crates/datadog-agent-config/mod.rs b/crates/datadog-agent-config/src/lib.rs similarity index 73% rename from crates/datadog-agent-config/mod.rs rename to crates/datadog-agent-config/src/lib.rs index 6fc858a..f671293 100644 --- a/crates/datadog-agent-config/mod.rs +++ b/crates/datadog-agent-config/src/lib.rs @@ -1,244 +1,45 @@ -pub mod additional_endpoints; -pub mod apm_replace_rule; -pub mod env; -pub mod flush_strategy; -pub mod log_level; -pub mod logs_additional_endpoints; -pub mod processing_rule; -pub mod service_mapping; -pub mod yaml; +pub mod deserializers; +pub mod sources; + +// Re-export submodules at the crate root so existing imports like +// `crate::flush_strategy::FlushStrategy` and `crate::env::EnvConfigSource` keep working. +pub use deserializers::{ + additional_endpoints, apm_replace_rule, flush_strategy, log_level, logs_additional_endpoints, + processing_rule, service_mapping, +}; +pub use sources::{env, yaml}; pub use datadog_opentelemetry::configuration::TracePropagationStyle; +// Re-export all helper deserializers so consumers and internal modules can +// use `crate::deserialize_optional_string` etc. without reaching into submodules. +pub use deserializers::helpers::*; use libdd_trace_obfuscation::replacer::ReplaceRule; use libdd_trace_utils::config_utils::{trace_intake_url, trace_intake_url_prefixed}; -use serde::{Deserialize, Deserializer}; -use serde_aux::prelude::deserialize_bool_from_anything; -use serde_json::Value; +use serde::Deserialize; +use std::collections::HashMap; use std::path::Path; -use std::time::Duration; -use std::{collections::HashMap, fmt}; -use tracing::{debug, error, warn}; +use tracing::{debug, error}; use crate::{ apm_replace_rule::deserialize_apm_replace_rules, env::EnvConfigSource, - flush_strategy::FlushStrategy, log_level::LogLevel, logs_additional_endpoints::LogsAdditionalEndpoint, processing_rule::{ProcessingRule, deserialize_processing_rules}, yaml::YamlConfigSource, }; -/// Helper macro to merge Option fields to String fields -/// -/// Providing one field argument will merge the value from the source config field into the config -/// field. -/// -/// Providing two field arguments will merge the value from the source config field into the config -/// field if the value is not empty. -#[macro_export] -macro_rules! merge_string { - ($config:expr, $config_field:ident, $source:expr, $source_field:ident) => { - if let Some(value) = &$source.$source_field { - $config.$config_field.clone_from(value); - } - }; - ($config:expr, $source:expr, $field:ident) => { - if let Some(value) = &$source.$field { - $config.$field.clone_from(value); - } - }; -} - -/// Helper macro to merge Option fields where T implements Clone -/// -/// Providing one field argument will merge the value from the source config field into the config -/// field. -/// -/// Providing two field arguments will merge the value from the source config field into the config -/// field if the value is not empty. -#[macro_export] -macro_rules! merge_option { - ($config:expr, $config_field:ident, $source:expr, $source_field:ident) => { - if $source.$source_field.is_some() { - $config.$config_field.clone_from(&$source.$source_field); - } - }; - ($config:expr, $source:expr, $field:ident) => { - if $source.$field.is_some() { - $config.$field.clone_from(&$source.$field); - } - }; -} - -/// Helper macro to merge Option fields to T fields when Option is Some -/// -/// Providing one field argument will merge the value from the source config field into the config -/// field. -/// -/// Providing two field arguments will merge the value from the source config field into the config -/// field if the value is not empty. -#[macro_export] -macro_rules! merge_option_to_value { - ($config:expr, $config_field:ident, $source:expr, $source_field:ident) => { - if let Some(value) = &$source.$source_field { - $config.$config_field = value.clone(); - } - }; - ($config:expr, $source:expr, $field:ident) => { - if let Some(value) = &$source.$field { - $config.$field = value.clone(); - } - }; -} - -/// Helper macro to merge `Vec` fields when `Vec` is not empty -/// -/// Providing one field argument will merge the value from the source config field into the config -/// field. -/// -/// Providing two field arguments will merge the value from the source config field into the config -/// field if the value is not empty. -#[macro_export] -macro_rules! merge_vec { - ($config:expr, $config_field:ident, $source:expr, $source_field:ident) => { - if !$source.$source_field.is_empty() { - $config.$config_field.clone_from(&$source.$source_field); - } - }; - ($config:expr, $source:expr, $field:ident) => { - if !$source.$field.is_empty() { - $config.$field.clone_from(&$source.$field); - } - }; -} - -// nit: these will replace one map with the other, not merge the maps togehter, right? -/// Helper macro to merge `HashMap` fields when `HashMap` is not empty -/// -/// Providing one field argument will merge the value from the source config field into the config -/// field. -/// -/// Providing two field arguments will merge the value from the source config field into the config -/// field if the value is not empty. -#[macro_export] -macro_rules! merge_hashmap { - ($config:expr, $config_field:ident, $source:expr, $source_field:ident) => { - if !$source.$source_field.is_empty() { - $config.$config_field.clone_from(&$source.$source_field); - } - }; - ($config:expr, $source:expr, $field:ident) => { - if !$source.$field.is_empty() { - $config.$field.clone_from(&$source.$field); - } - }; -} - -#[derive(Debug, PartialEq)] -#[allow(clippy::module_name_repetitions)] -pub enum ConfigError { - ParseError(String), - UnsupportedField(String), -} - -#[allow(clippy::module_name_repetitions)] -pub trait ConfigSource { - fn load(&self, config: &mut Config) -> Result<(), ConfigError>; -} - -#[derive(Default)] -#[allow(clippy::module_name_repetitions)] -pub struct ConfigBuilder { - sources: Vec>, - config: Config, -} - -#[allow(clippy::module_name_repetitions)] -impl ConfigBuilder { - #[must_use] - pub fn add_source(mut self, source: Box) -> Self { - self.sources.push(source); - self - } - - pub fn build(&mut self) -> Config { - let mut failed_sources = 0; - for source in &self.sources { - match source.load(&mut self.config) { - Ok(()) => (), - Err(e) => { - error!("Failed to load config: {:?}", e); - failed_sources += 1; - } - } - } - - if !self.sources.is_empty() && failed_sources == self.sources.len() { - debug!("All sources failed to load config, using default config."); - } - - if self.config.site.is_empty() { - self.config.site = "datadoghq.com".to_string(); - } - - // If `proxy_https` is not set, set it from `HTTPS_PROXY` environment variable - // if it exists - if let Ok(https_proxy) = std::env::var("HTTPS_PROXY") - && self.config.proxy_https.is_none() - { - self.config.proxy_https = Some(https_proxy); - } - - // If `proxy_https` is set, check if the site is in `NO_PROXY` environment variable - // or in the `proxy_no_proxy` config field. - if self.config.proxy_https.is_some() { - let site_in_no_proxy = std::env::var("NO_PROXY") - .is_ok_and(|no_proxy| no_proxy.contains(&self.config.site)) - || self - .config - .proxy_no_proxy - .iter() - .any(|no_proxy| no_proxy.contains(&self.config.site)); - if site_in_no_proxy { - self.config.proxy_https = None; - } - } - - // If extraction is not set, set it to the same as the propagation style - if self.config.trace_propagation_style_extract.is_empty() { - self.config - .trace_propagation_style_extract - .clone_from(&self.config.trace_propagation_style); - } - - // If Logs URL is not set, set it to the default - if self.config.logs_config_logs_dd_url.trim().is_empty() { - self.config.logs_config_logs_dd_url = build_fqdn_logs(self.config.site.clone()); - } else { - self.config.logs_config_logs_dd_url = - logs_intake_url(self.config.logs_config_logs_dd_url.as_str()); - } - - // If APM URL is not set, set it to the default - if self.config.apm_dd_url.is_empty() { - self.config.apm_dd_url = trace_intake_url(self.config.site.clone().as_str()); - } else { - // If APM URL is set, add the site to the URL - self.config.apm_dd_url = trace_intake_url_prefixed(self.config.apm_dd_url.as_str()); - } - - self.config.clone() - } -} +// --------------------------------------------------------------------------- +// Config — the resolved configuration struct +// --------------------------------------------------------------------------- #[derive(Debug, PartialEq, Clone)] #[allow(clippy::module_name_repetitions)] #[allow(clippy::struct_excessive_bools)] -pub struct Config { +pub struct Config { pub site: String, pub api_key: String, pub log_level: LogLevel, @@ -349,28 +150,12 @@ pub struct Config { // - Logs pub otlp_config_logs_enabled: bool, - // AWS Lambda - pub api_key_secret_arn: String, - pub kms_api_key: String, - pub api_key_ssm_arn: String, - pub serverless_logs_enabled: bool, - pub serverless_flush_strategy: FlushStrategy, - pub enhanced_metrics: bool, - pub lambda_proc_enhanced_metrics: bool, - pub capture_lambda_payload: bool, - pub capture_lambda_payload_max_depth: u32, - pub compute_trace_stats_on_extension: bool, - pub span_dedup_timeout: Option, - pub api_key_secret_reload_interval: Option, - - pub serverless_appsec_enabled: bool, - pub appsec_rules: Option, - pub appsec_waf_timeout: Duration, - pub api_security_enabled: bool, - pub api_security_sample_delay: Duration, + /// Agent-specific extension fields defined by the consumer. + /// Use `NoExtension` (the default) when no extra fields are needed. + pub ext: E, } -impl Default for Config { +impl Default for Config { fn default() -> Self { Self { site: String::default(), @@ -464,33 +249,30 @@ impl Default for Config { otlp_config_traces_probabilistic_sampler_sampling_percentage: None, otlp_config_logs_enabled: false, - // AWS Lambda - api_key_secret_arn: String::default(), - kms_api_key: String::default(), - api_key_ssm_arn: String::default(), - serverless_logs_enabled: true, - serverless_flush_strategy: FlushStrategy::Default, - enhanced_metrics: true, - lambda_proc_enhanced_metrics: true, - capture_lambda_payload: false, - capture_lambda_payload_max_depth: 10, - compute_trace_stats_on_extension: false, - span_dedup_timeout: None, - api_key_secret_reload_interval: None, - - serverless_appsec_enabled: false, - appsec_rules: None, - appsec_waf_timeout: Duration::from_millis(5), - api_security_enabled: true, - api_security_sample_delay: Duration::from_secs(30), + ext: E::default(), } } } +// --------------------------------------------------------------------------- +// Loading — entry points for building a Config +// --------------------------------------------------------------------------- + #[allow(clippy::module_name_repetitions)] #[inline] #[must_use] pub fn get_config(config_directory: &Path) -> Config { + get_config_with_extension(config_directory) +} + +/// Load configuration with a custom extension type. +/// +/// Consumers that need additional fields should call this with their +/// extension type instead of `get_config`. +#[allow(clippy::module_name_repetitions)] +#[inline] +#[must_use] +pub fn get_config_with_extension(config_directory: &Path) -> Config { let path: std::path::PathBuf = config_directory.join("datadog.yaml"); ConfigBuilder::default() .add_source(Box::new(YamlConfigSource { path })) @@ -498,400 +280,366 @@ pub fn get_config(config_directory: &Path) -> Config { .build() } -#[inline] -#[must_use] -fn build_fqdn_logs(site: String) -> String { - format!("https://http-intake.logs.{site}") +// --------------------------------------------------------------------------- +// ConfigExtension — trait for additional configuration fields +// --------------------------------------------------------------------------- + +/// Trait that extension configs must implement to add additional configuration +/// fields beyond what the core provides. +/// +/// Extensions allow consumers to define their own external configuration fields +/// that are deserialized from environment variables and YAML files alongside +/// core fields via dual extraction. +/// +/// # Source type requirements +/// +/// The `Source` type **must** use `#[serde(default)]` on the struct and graceful +/// deserializers (e.g., `deserialize_optional_bool_from_anything`) on each field. +/// Without these, a missing or malformed value will cause the entire extension +/// extraction to fail — the extension silently falls back to `E::default()` with +/// a `tracing::warn!` log. See [`ConfigExtension::Source`] for details. +/// +/// # Flat fields only +/// +/// A single `Source` type is used for both environment variable and YAML +/// extraction. This works when all extension fields are top-level (flat) in +/// the YAML file, which is the common case for extension configs: +/// +/// ```yaml +/// # Works: flat fields map naturally to both DD_* env vars and YAML keys +/// enhanced_metrics: true +/// capture_lambda_payload: false +/// ``` +/// +/// If you need nested YAML structures (e.g., `lambda: { enhanced_metrics: true }`) +/// that differ from the flat env var layout, implement `merge_from` with a +/// nested source struct and handle the mapping manually instead of using +/// `merge_fields!`. +/// +/// # Field name collisions with core config +/// +/// Extension fields are extracted independently from the same figment as core +/// fields. If an extension defines a field with the same name as a core field +/// (e.g., `api_key`), both will deserialize their own copy — they do not +/// interfere with each other, but the extension copy will **not** override the +/// core value. Avoid shadowing core field names to prevent confusion. +pub trait ConfigExtension: Clone + Default + std::fmt::Debug + PartialEq { + /// Intermediate deserialization type for extension fields, used for both + /// environment variable and YAML extraction. + /// + /// # Requirements + /// + /// The struct **must** have: + /// + /// 1. `#[serde(default)]` on the struct — so missing fields get defaults + /// instead of failing the whole extraction. + /// 2. Graceful per-field deserializers (e.g., + /// `#[serde(deserialize_with = "deserialize_optional_bool_from_anything")]`) + /// — so one malformed value doesn't fail the whole extraction. + /// + /// **If either is missing**, `figment::extract::()` will fail at + /// runtime when a field is absent or malformed. The extension falls back to + /// `E::default()` and a `tracing::warn!` is emitted — no panic, but all + /// extension fields silently get their default values. + type Source: Default + serde::de::DeserializeOwned + Clone + std::fmt::Debug; + + /// Merge parsed source fields into self. + fn merge_from(&mut self, source: &Self::Source); } -#[inline] -#[must_use] -fn logs_intake_url(url: &str) -> String { - let url = url.trim(); - if url.is_empty() { - return url.to_string(); - } - if url.starts_with("https://") || url.starts_with("http://") { - return url.to_string(); - } - format!("https://{url}") +/// A no-op extension for consumers that don't need extra fields. +#[derive(Clone, Default, Debug, PartialEq)] +pub struct NoExtension; + +/// A no-op source for deserialization that accepts (and ignores) any input. +/// Uses a regular struct (not unit struct) so serde deserializes it from +/// map-shaped data that figment provides, rather than expecting null/unit. +#[derive(Clone, Default, Debug, Deserialize)] +pub struct NoExtensionSource {} + +impl ConfigExtension for NoExtension { + type Source = NoExtensionSource; + fn merge_from(&mut self, _source: &Self::Source) {} } -pub fn deserialize_optional_string<'de, D>(deserializer: D) -> Result, D::Error> -where - D: Deserializer<'de>, -{ - match Value::deserialize(deserializer)? { - Value::String(s) => Ok(Some(s)), - other => { - warn!( - "Failed to parse value, expected a string, got: {}, ignoring", - other - ); - Ok(None) - } - } +// --------------------------------------------------------------------------- +// ConfigBuilder — orchestrates loading from multiple sources +// --------------------------------------------------------------------------- + +#[derive(Debug, PartialEq)] +#[allow(clippy::module_name_repetitions)] +pub enum ConfigError { + ParseError(String), + UnsupportedField(String), } -pub fn deserialize_string_or_int<'de, D>(deserializer: D) -> Result, D::Error> -where - D: Deserializer<'de>, -{ - let value = Value::deserialize(deserializer)?; - match value { - Value::String(s) => { - if s.trim().is_empty() { - Ok(None) - } else { - Ok(Some(s)) - } - } - Value::Number(n) => Ok(Some(n.to_string())), - _ => { - warn!("Failed to parse value, expected a string or an integer, ignoring"); - Ok(None) - } - } +#[allow(clippy::module_name_repetitions)] +pub trait ConfigSource { + fn load(&self, config: &mut Config) -> Result<(), ConfigError>; } -pub fn deserialize_optional_bool_from_anything<'de, D>( - deserializer: D, -) -> Result, D::Error> -where - D: Deserializer<'de>, -{ - // First try to deserialize as Option<_> to handle null/missing values - let opt: Option = Option::deserialize(deserializer)?; - - match opt { - None => Ok(None), - Some(value) => match deserialize_bool_from_anything(value) { - Ok(bool_result) => Ok(Some(bool_result)), - Err(e) => { - warn!("Failed to parse bool value: {}, ignoring", e); - Ok(None) - } - }, - } +#[allow(clippy::module_name_repetitions)] +pub struct ConfigBuilder { + sources: Vec>>, + config: Config, } -/// Parse a single "key:value" string into a (key, value) tuple -/// Returns None if the string is invalid (e.g., missing colon, empty key/value) -fn parse_key_value_tag(tag: &str) -> Option<(String, String)> { - let parts: Vec<&str> = tag.splitn(2, ':').collect(); - if parts.len() == 2 && !parts[0].is_empty() && !parts[1].is_empty() { - Some((parts[0].to_string(), parts[1].to_string())) - } else { - warn!( - "Failed to parse tag '{}', expected format 'key:value', ignoring", - tag - ); - None +impl Default for ConfigBuilder { + fn default() -> Self { + Self { + sources: Vec::new(), + config: Config::default(), + } } } -pub fn deserialize_key_value_pairs<'de, D>( - deserializer: D, -) -> Result, D::Error> -where - D: Deserializer<'de>, -{ - struct KeyValueVisitor; - - impl serde::de::Visitor<'_> for KeyValueVisitor { - type Value = HashMap; - - fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result { - formatter.write_str("a string in format 'key1:value1,key2:value2' or 'key1:value1'") - } +#[allow(clippy::module_name_repetitions)] +impl ConfigBuilder { + #[must_use] + pub fn add_source(mut self, source: Box>) -> Self { + self.sources.push(source); + self + } - fn visit_str(self, value: &str) -> Result - where - E: serde::de::Error, - { - let mut map = HashMap::new(); - for tag in value.split(&[',', ' ']) { - if tag.is_empty() { - continue; - } - if let Some((key, val)) = parse_key_value_tag(tag) { - map.insert(key, val); + pub fn build(&mut self) -> Config { + let mut failed_sources = 0; + for source in &self.sources { + match source.load(&mut self.config) { + Ok(()) => (), + Err(e) => { + error!("Failed to load config: {:?}", e); + failed_sources += 1; } } + } - Ok(map) + if !self.sources.is_empty() && failed_sources == self.sources.len() { + debug!("All sources failed to load config, using default config."); } - fn visit_u64(self, value: u64) -> Result - where - E: serde::de::Error, - { - warn!( - "Failed to parse tags: expected string in format 'key:value', got number {}, ignoring", - value - ); - Ok(HashMap::new()) + if self.config.site.is_empty() { + self.config.site = "datadoghq.com".to_string(); } - fn visit_i64(self, value: i64) -> Result - where - E: serde::de::Error, + // If `proxy_https` is not set, set it from `HTTPS_PROXY` environment variable + // if it exists + if let Ok(https_proxy) = std::env::var("HTTPS_PROXY") + && self.config.proxy_https.is_none() { - warn!( - "Failed to parse tags: expected string in format 'key:value', got number {}, ignoring", - value - ); - Ok(HashMap::new()) + self.config.proxy_https = Some(https_proxy); } - fn visit_f64(self, value: f64) -> Result - where - E: serde::de::Error, - { - warn!( - "Failed to parse tags: expected string in format 'key:value', got number {}, ignoring", - value - ); - Ok(HashMap::new()) + // If `proxy_https` is set, check if the site is in `NO_PROXY` environment variable + // or in the `proxy_no_proxy` config field. + if self.config.proxy_https.is_some() { + let site_in_no_proxy = std::env::var("NO_PROXY") + .is_ok_and(|no_proxy| no_proxy.contains(&self.config.site)) + || self + .config + .proxy_no_proxy + .iter() + .any(|no_proxy| no_proxy.contains(&self.config.site)); + if site_in_no_proxy { + self.config.proxy_https = None; + } + } + + // If extraction is not set, set it to the same as the propagation style + if self.config.trace_propagation_style_extract.is_empty() { + self.config + .trace_propagation_style_extract + .clone_from(&self.config.trace_propagation_style); + } + + // If Logs URL is not set, set it to the default + if self.config.logs_config_logs_dd_url.trim().is_empty() { + self.config.logs_config_logs_dd_url = build_fqdn_logs(self.config.site.clone()); + } else { + self.config.logs_config_logs_dd_url = + logs_intake_url(self.config.logs_config_logs_dd_url.as_str()); } - fn visit_bool(self, value: bool) -> Result - where - E: serde::de::Error, - { - warn!( - "Failed to parse tags: expected string in format 'key:value', got boolean {}, ignoring", - value - ); - Ok(HashMap::new()) + // If APM URL is not set, set it to the default + if self.config.apm_dd_url.is_empty() { + self.config.apm_dd_url = trace_intake_url(self.config.site.clone().as_str()); + } else { + // If APM URL is set, add the site to the URL + self.config.apm_dd_url = trace_intake_url_prefixed(self.config.apm_dd_url.as_str()); } + + self.config.clone() } +} - deserializer.deserialize_any(KeyValueVisitor) +#[inline] +#[must_use] +fn build_fqdn_logs(site: String) -> String { + format!("https://http-intake.logs.{site}") } -pub fn deserialize_array_from_comma_separated_string<'de, D>( - deserializer: D, -) -> Result, D::Error> -where - D: Deserializer<'de>, -{ - let s: String = String::deserialize(deserializer)?; - Ok(s.split(',') - .map(|feature| feature.trim().to_string()) - .filter(|feature| !feature.is_empty()) - .collect()) +#[inline] +#[must_use] +fn logs_intake_url(url: &str) -> String { + let url = url.trim(); + if url.is_empty() { + return url.to_string(); + } + if url.starts_with("https://") || url.starts_with("http://") { + return url.to_string(); + } + format!("https://{url}") } -pub fn deserialize_key_value_pair_array_to_hashmap<'de, D>( - deserializer: D, -) -> Result, D::Error> -where - D: Deserializer<'de>, -{ - let array: Vec = match Vec::deserialize(deserializer) { - Ok(v) => v, - Err(e) => { - warn!("Failed to deserialize tags array: {e}, ignoring"); - return Ok(HashMap::new()); +// --------------------------------------------------------------------------- +// Merge macros — used by sources and extension implementations +// --------------------------------------------------------------------------- + +/// Helper macro to merge Option fields to String fields +/// +/// Providing one field argument will merge the value from the source config field into the config +/// field. +/// +/// Providing two field arguments will merge the value from the source config field into the config +/// field if the value is not empty. +#[macro_export] +macro_rules! merge_string { + ($config:expr, $config_field:ident, $source:expr, $source_field:ident) => { + if let Some(value) = &$source.$source_field { + $config.$config_field.clone_from(value); } }; - let mut map = HashMap::new(); - for s in array { - if let Some((key, val)) = parse_key_value_tag(&s) { - map.insert(key, val); + ($config:expr, $source:expr, $field:ident) => { + if let Some(value) = &$source.$field { + $config.$field.clone_from(value); } - } - Ok(map) + }; } -/// Deserialize APM filter tags from space-separated "key:value" pairs, also support key-only tags -pub fn deserialize_apm_filter_tags<'de, D>(deserializer: D) -> Result>, D::Error> -where - D: Deserializer<'de>, -{ - let opt: Option = Option::deserialize(deserializer)?; - - match opt { - None => Ok(None), - Some(s) if s.trim().is_empty() => Ok(None), - Some(s) => { - let tags: Vec = s - .split_whitespace() - .filter_map(|pair| { - let parts: Vec<&str> = pair.splitn(2, ':').collect(); - if parts.len() == 2 { - let key = parts[0].trim(); - let value = parts[1].trim(); - if key.is_empty() { - None - } else if value.is_empty() { - Some(key.to_string()) - } else { - Some(format!("{key}:{value}")) - } - } else if parts.len() == 1 { - let key = parts[0].trim(); - if key.is_empty() { - None - } else { - Some(key.to_string()) - } - } else { - None - } - }) - .collect(); - - if tags.is_empty() { - Ok(None) - } else { - Ok(Some(tags)) - } +/// Helper macro to merge Option fields where T implements Clone +/// +/// Providing one field argument will merge the value from the source config field into the config +/// field. +/// +/// Providing two field arguments will merge the value from the source config field into the config +/// field if the value is not empty. +#[macro_export] +macro_rules! merge_option { + ($config:expr, $config_field:ident, $source:expr, $source_field:ident) => { + if $source.$source_field.is_some() { + $config.$config_field.clone_from(&$source.$source_field); } - } -} - -pub fn deserialize_option_lossless<'de, D, T>(deserializer: D) -> Result, D::Error> -where - D: Deserializer<'de>, - T: Deserialize<'de>, -{ - match Option::::deserialize(deserializer) { - Ok(value) => Ok(value), - Err(e) => { - warn!("Failed to deserialize optional value: {}, ignoring", e); - Ok(None) + }; + ($config:expr, $source:expr, $field:ident) => { + if $source.$field.is_some() { + $config.$field.clone_from(&$source.$field); } - } + }; } -/// Gracefully deserialize any field, falling back to `T::default()` on error. +/// Helper macro to merge Option fields to T fields when Option is Some +/// +/// Providing one field argument will merge the value from the source config field into the config +/// field. /// -/// This ensures that a single field with the wrong type never fails the entire -/// struct extraction. Works for any `T` that implements `Deserialize + Default`: -/// - `Option` defaults to `None` -/// - `Vec` defaults to `[]` -/// - `HashMap` defaults to `{}` -/// - Structs with `#[derive(Default)]` use their default -pub fn deserialize_with_default<'de, D, T>(deserializer: D) -> Result -where - D: Deserializer<'de>, - T: Deserialize<'de> + Default, -{ - match T::deserialize(deserializer) { - Ok(value) => Ok(value), - Err(e) => { - warn!("Failed to deserialize field: {}, using default", e); - Ok(T::default()) +/// Providing two field arguments will merge the value from the source config field into the config +/// field if the value is not empty. +#[macro_export] +macro_rules! merge_option_to_value { + ($config:expr, $config_field:ident, $source:expr, $source_field:ident) => { + if let Some(value) = &$source.$source_field { + $config.$config_field = value.clone(); } - } -} - -pub fn deserialize_optional_duration_from_microseconds<'de, D: Deserializer<'de>>( - deserializer: D, -) -> Result, D::Error> { - match Option::::deserialize(deserializer) { - Ok(opt) => Ok(opt.map(Duration::from_micros)), - Err(e) => { - warn!("Failed to deserialize duration (microseconds): {e}, ignoring"); - Ok(None) + }; + ($config:expr, $source:expr, $field:ident) => { + if let Some(value) = &$source.$field { + $config.$field = value.clone(); } - } + }; } -pub fn deserialize_optional_duration_from_seconds<'de, D: Deserializer<'de>>( - deserializer: D, -) -> Result, D::Error> { - // Deserialize into a generic Value first to avoid propagating type errors, - // then try to extract a duration from it. - match Value::deserialize(deserializer) { - Ok(Value::Number(n)) => { - if let Some(u) = n.as_u64() { - Ok(Some(Duration::from_secs(u))) - } else if let Some(i) = n.as_i64() { - if i < 0 { - warn!("Failed to parse duration: negative durations are not allowed, ignoring"); - Ok(None) - } else { - Ok(Some(Duration::from_secs(i as u64))) - } - } else if let Some(f) = n.as_f64() { - if f < 0.0 { - warn!("Failed to parse duration: negative durations are not allowed, ignoring"); - Ok(None) - } else { - Ok(Some(Duration::from_secs_f64(f))) - } - } else { - warn!("Failed to parse duration: unsupported number format, ignoring"); - Ok(None) - } - } - Ok(Value::Null) => Ok(None), - Ok(other) => { - warn!("Failed to parse duration: expected number, got {other}, ignoring"); - Ok(None) +/// Helper macro to merge `Vec` fields when `Vec` is not empty +/// +/// Providing one field argument will merge the value from the source config field into the config +/// field. +/// +/// Providing two field arguments will merge the value from the source config field into the config +/// field if the value is not empty. +#[macro_export] +macro_rules! merge_vec { + ($config:expr, $config_field:ident, $source:expr, $source_field:ident) => { + if !$source.$source_field.is_empty() { + $config.$config_field.clone_from(&$source.$source_field); } - Err(e) => { - warn!("Failed to deserialize duration: {e}, ignoring"); - Ok(None) + }; + ($config:expr, $source:expr, $field:ident) => { + if !$source.$field.is_empty() { + $config.$field.clone_from(&$source.$field); } - } -} - -// Like deserialize_optional_duration_from_seconds(), but return None if the value is 0 -pub fn deserialize_optional_duration_from_seconds_ignore_zero<'de, D: Deserializer<'de>>( - deserializer: D, -) -> Result, D::Error> { - let duration: Option = deserialize_optional_duration_from_seconds(deserializer)?; - if duration.is_some_and(|d| d.as_secs() == 0) { - return Ok(None); - } - Ok(duration) + }; } -pub fn deserialize_trace_propagation_style<'de, D>( - deserializer: D, -) -> Result, D::Error> -where - D: Deserializer<'de>, -{ - use std::str::FromStr; - let s: String = match String::deserialize(deserializer) { - Ok(s) => s, - Err(e) => { - warn!("Failed to deserialize trace propagation style: {e}, ignoring"); - return Ok(Vec::new()); +/// Helper macro to merge `HashMap` fields when `HashMap` is not empty +/// +/// Providing one field argument will merge the value from the source config field into the config +/// field. +/// +/// Providing two field arguments will merge the value from the source config field into the config +/// field if the value is not empty. +#[macro_export] +macro_rules! merge_hashmap { + ($config:expr, $config_field:ident, $source:expr, $source_field:ident) => { + if !$source.$source_field.is_empty() { + $config.$config_field.clone_from(&$source.$source_field); + } + }; + ($config:expr, $source:expr, $field:ident) => { + if !$source.$field.is_empty() { + $config.$field.clone_from(&$source.$field); } }; +} - Ok(s.split(',') - .filter_map( - |style| match TracePropagationStyle::from_str(style.trim()) { - Ok(parsed_style) => Some(parsed_style), - Err(e) => { - warn!("Failed to parse trace propagation style: {e}, ignoring"); - None - } - }, - ) - .collect()) +/// Batch-merge extension fields from a source struct. +/// +/// Groups fields by merge strategy so you don't have to write individual +/// `merge_string!` / `merge_option_to_value!` / `merge_option!` calls. +/// +/// ```ignore +/// merge_fields!(self, source, +/// string: [api_key_secret_arn, kms_api_key], +/// value: [enhanced_metrics, capture_lambda_payload], +/// option: [span_dedup_timeout, appsec_rules], +/// ); +/// ``` +#[macro_export] +macro_rules! merge_fields { + // Internal rules dispatched by keyword + (@string $config:expr, $source:expr, [$($field:ident),* $(,)?]) => { + $( $crate::merge_string!($config, $source, $field); )* + }; + (@value $config:expr, $source:expr, [$($field:ident),* $(,)?]) => { + $( $crate::merge_option_to_value!($config, $source, $field); )* + }; + (@option $config:expr, $source:expr, [$($field:ident),* $(,)?]) => { + $( $crate::merge_option!($config, $source, $field); )* + }; + // Public entry point: accepts any combination of groups in any order + ($config:expr, $source:expr, $($kind:ident: [$($field:ident),* $(,)?]),* $(,)?) => { + $( $crate::merge_fields!(@$kind $config, $source, [$($field),*]); )* + }; } #[cfg_attr(coverage_nightly, coverage(off))] // Test modules skew coverage metrics #[cfg(test)] +#[allow(clippy::result_large_err)] pub mod tests { use libdd_trace_obfuscation::replacer::parse_rules_from_string; use super::*; - use crate::{ - TracePropagationStyle, - flush_strategy::{FlushStrategy, PeriodicStrategy}, - log_level::LogLevel, - processing_rule::ProcessingRule, - }; + use std::time::Duration; + + use crate::{TracePropagationStyle, log_level::LogLevel, processing_rule::ProcessingRule}; #[test] fn test_default_logs_intake_url() { @@ -1158,56 +906,6 @@ pub mod tests { }); } - #[test] - fn test_parse_flush_strategy_end() { - figment::Jail::expect_with(|jail| { - jail.clear_env(); - jail.set_env("DD_SERVERLESS_FLUSH_STRATEGY", "end"); - let config = get_config(Path::new("")); - assert_eq!(config.serverless_flush_strategy, FlushStrategy::End); - Ok(()) - }); - } - - #[test] - fn test_parse_flush_strategy_periodically() { - figment::Jail::expect_with(|jail| { - jail.clear_env(); - jail.set_env("DD_SERVERLESS_FLUSH_STRATEGY", "periodically,100000"); - let config = get_config(Path::new("")); - assert_eq!( - config.serverless_flush_strategy, - FlushStrategy::Periodically(PeriodicStrategy { interval: 100_000 }) - ); - Ok(()) - }); - } - - #[test] - fn test_parse_flush_strategy_invalid() { - figment::Jail::expect_with(|jail| { - jail.clear_env(); - jail.set_env("DD_SERVERLESS_FLUSH_STRATEGY", "invalid_strategy"); - let config = get_config(Path::new("")); - assert_eq!(config.serverless_flush_strategy, FlushStrategy::Default); - Ok(()) - }); - } - - #[test] - fn test_parse_flush_strategy_invalid_periodic() { - figment::Jail::expect_with(|jail| { - jail.clear_env(); - jail.set_env( - "DD_SERVERLESS_FLUSH_STRATEGY", - "periodically,invalid_interval", - ); - let config = get_config(Path::new("")); - assert_eq!(config.serverless_flush_strategy, FlushStrategy::Default); - Ok(()) - }); - } - #[test] fn parse_number_or_string_env_vars() { figment::Jail::expect_with(|jail| { @@ -1476,15 +1174,11 @@ pub mod tests { fn test_parse_bool_from_anything() { figment::Jail::expect_with(|jail| { jail.clear_env(); - jail.set_env("DD_SERVERLESS_LOGS_ENABLED", "true"); - jail.set_env("DD_ENHANCED_METRICS", "1"); jail.set_env("DD_LOGS_CONFIG_USE_COMPRESSION", "TRUE"); - jail.set_env("DD_CAPTURE_LAMBDA_PAYLOAD", "0"); + jail.set_env("DD_SKIP_SSL_VALIDATION", "1"); let config = get_config(Path::new("")); - assert!(config.serverless_logs_enabled); - assert!(config.enhanced_metrics); assert!(config.logs_config_use_compression); - assert!(!config.capture_lambda_payload); + assert!(config.skip_ssl_validation); Ok(()) }); } @@ -1708,4 +1402,144 @@ pub mod tests { serde_json::from_str::(r#"{"tags": []}"#).expect("failed to parse JSON"); assert_eq!(result.tags, HashMap::new()); } + + // -- ConfigExtension tests -- + + /// A test extension with a few fields, mimicking what a consumer like Lambda would define. + #[derive(Clone, Default, Debug, PartialEq)] + struct TestExtension { + custom_flag: bool, + custom_name: String, + } + + #[derive(Clone, Default, Debug, Deserialize)] + #[serde(default)] + struct TestExtSource { + #[serde(deserialize_with = "deserialize_optional_bool_from_anything")] + custom_flag: Option, + #[serde(deserialize_with = "deserialize_optional_string")] + custom_name: Option, + } + + impl ConfigExtension for TestExtension { + type Source = TestExtSource; + + fn merge_from(&mut self, source: &TestExtSource) { + merge_fields!(self, source, + string: [custom_name], + value: [custom_flag], + ); + } + } + + #[test] + fn test_no_extension_config_works() { + figment::Jail::expect_with(|jail| { + jail.clear_env(); + jail.set_env("DD_SITE", "datad0g.com"); + let config = get_config(Path::new("")); + assert_eq!(config.site, "datad0g.com"); + assert_eq!(config.ext, NoExtension); + Ok(()) + }); + } + + #[test] + fn test_extension_receives_env_vars() { + figment::Jail::expect_with(|jail| { + jail.clear_env(); + jail.set_env("DD_SITE", "datad0g.com"); + jail.set_env("DD_CUSTOM_FLAG", "true"); + jail.set_env("DD_CUSTOM_NAME", "my-extension"); + + let config: Config = get_config_with_extension(Path::new("")); + + // Core fields work + assert_eq!(config.site, "datad0g.com"); + // Extension fields are populated + assert!(config.ext.custom_flag); + assert_eq!(config.ext.custom_name, "my-extension"); + Ok(()) + }); + } + + #[test] + fn test_extension_receives_yaml_fields() { + figment::Jail::expect_with(|jail| { + jail.clear_env(); + jail.create_file( + "datadog.yaml", + r#" +site: "datad0g.com" +custom_flag: true +custom_name: "yaml-ext" +"#, + )?; + + let config: Config = get_config_with_extension(Path::new("")); + + assert_eq!(config.site, "datad0g.com"); + assert!(config.ext.custom_flag); + assert_eq!(config.ext.custom_name, "yaml-ext"); + Ok(()) + }); + } + + #[test] + fn test_extension_env_overrides_yaml() { + figment::Jail::expect_with(|jail| { + jail.clear_env(); + jail.create_file( + "datadog.yaml", + r#" +custom_name: "yaml-value" +custom_flag: false +"#, + )?; + jail.set_env("DD_CUSTOM_NAME", "env-value"); + jail.set_env("DD_CUSTOM_FLAG", "true"); + + let config: Config = get_config_with_extension(Path::new("")); + + // Env should override YAML (env source loaded after yaml) + assert!(config.ext.custom_flag); + assert_eq!(config.ext.custom_name, "env-value"); + Ok(()) + }); + } + + #[test] + fn test_extension_defaults_when_not_set() { + figment::Jail::expect_with(|jail| { + jail.clear_env(); + + let config: Config = get_config_with_extension(Path::new("")); + + // Extension fields should be at their defaults + assert!(!config.ext.custom_flag); + assert_eq!(config.ext.custom_name, ""); + // Core fields should have post-processing defaults + assert_eq!(config.site, "datadoghq.com"); + Ok(()) + }); + } + + #[test] + fn test_extension_does_not_interfere_with_core() { + figment::Jail::expect_with(|jail| { + jail.clear_env(); + jail.set_env("DD_SITE", "us5.datadoghq.com"); + jail.set_env("DD_API_KEY", "test-key"); + jail.set_env("DD_CUSTOM_FLAG", "true"); + + let config: Config = get_config_with_extension(Path::new("")); + + // Core fields are not affected by extension env vars + assert_eq!(config.site, "us5.datadoghq.com"); + assert_eq!(config.api_key, "test-key"); + // Extension fields work alongside core + assert!(config.ext.custom_flag); + Ok(()) + }); + } } diff --git a/crates/datadog-agent-config/env.rs b/crates/datadog-agent-config/src/sources/env.rs similarity index 75% rename from crates/datadog-agent-config/env.rs rename to crates/datadog-agent-config/src/sources/env.rs index f24d6be..6fb96a1 100644 --- a/crates/datadog-agent-config/env.rs +++ b/crates/datadog-agent-config/src/sources/env.rs @@ -1,22 +1,18 @@ use figment::{Figment, providers::Env}; use serde::Deserialize; use std::collections::HashMap; -use std::time::Duration; use dogstatsd::util::parse_metric_namespace; use libdd_trace_obfuscation::replacer::ReplaceRule; use crate::{ - Config, ConfigError, ConfigSource, TracePropagationStyle, + Config, ConfigError, ConfigExtension, ConfigSource, TracePropagationStyle, additional_endpoints::deserialize_additional_endpoints, apm_replace_rule::deserialize_apm_replace_rules, deserialize_apm_filter_tags, deserialize_array_from_comma_separated_string, deserialize_key_value_pairs, deserialize_option_lossless, - deserialize_optional_bool_from_anything, deserialize_optional_duration_from_microseconds, - deserialize_optional_duration_from_seconds, - deserialize_optional_duration_from_seconds_ignore_zero, deserialize_optional_string, + deserialize_optional_bool_from_anything, deserialize_optional_string, deserialize_string_or_int, deserialize_trace_propagation_style, deserialize_with_default, - flush_strategy::FlushStrategy, log_level::LogLevel, logs_additional_endpoints::{LogsAdditionalEndpoint, deserialize_logs_additional_endpoints}, merge_hashmap, merge_option, merge_option_to_value, merge_string, merge_vec, @@ -369,119 +365,10 @@ pub struct EnvConfig { /// @env `DD_OTLP_CONFIG_LOGS_ENABLED` #[serde(deserialize_with = "deserialize_optional_bool_from_anything")] pub otlp_config_logs_enabled: Option, - - // AWS Lambda - /// @env `DD_API_KEY_SECRET_ARN` - /// - /// The AWS ARN of the secret containing the Datadog API key. - #[serde(deserialize_with = "deserialize_optional_string")] - pub api_key_secret_arn: Option, - /// @env `DD_KMS_API_KEY` - /// - /// The AWS KMS API key to use for the Datadog Agent. - #[serde(deserialize_with = "deserialize_optional_string")] - pub kms_api_key: Option, - /// @env `DD_API_KEY_SSM_ARN` - /// - /// The AWS Systems Manager Parameter Store parameter ARN containing the Datadog API key. - #[serde(deserialize_with = "deserialize_optional_string")] - pub api_key_ssm_arn: Option, - /// @env `DD_SERVERLESS_LOGS_ENABLED` - /// - /// Enable logs for AWS Lambda. Default is `true`. - #[serde(deserialize_with = "deserialize_optional_bool_from_anything")] - pub serverless_logs_enabled: Option, - /// @env `DD_LOGS_ENABLED` - /// - /// Enable logs for AWS Lambda. Alias for `DD_SERVERLESS_LOGS_ENABLED`. Default is `true`. - #[serde(deserialize_with = "deserialize_optional_bool_from_anything")] - pub logs_enabled: Option, - /// @env `DD_SERVERLESS_FLUSH_STRATEGY` - /// - /// The flush strategy to use for AWS Lambda. - #[serde(deserialize_with = "deserialize_with_default")] - pub serverless_flush_strategy: Option, - /// @env `DD_ENHANCED_METRICS` - /// - /// Enable enhanced metrics for AWS Lambda. Default is `true`. - #[serde(deserialize_with = "deserialize_optional_bool_from_anything")] - pub enhanced_metrics: Option, - /// @env `DD_LAMBDA_PROC_ENHANCED_METRICS` - /// - /// Enable Lambda process metrics for AWS Lambda. Default is `true`. - /// - /// This is for metrics like: - /// - CPU usage - /// - Network usage - /// - File descriptor count - /// - Thread count - /// - Temp directory usage - #[serde(deserialize_with = "deserialize_optional_bool_from_anything")] - pub lambda_proc_enhanced_metrics: Option, - /// @env `DD_CAPTURE_LAMBDA_PAYLOAD` - /// - /// Enable capture of the Lambda request and response payloads. - /// Default is `false`. - #[serde(deserialize_with = "deserialize_optional_bool_from_anything")] - pub capture_lambda_payload: Option, - /// @env `DD_CAPTURE_LAMBDA_PAYLOAD_MAX_DEPTH` - /// - /// The maximum depth of the Lambda payload to capture. - /// Default is `10`. Requires `capture_lambda_payload` to be `true`. - #[serde(deserialize_with = "deserialize_option_lossless")] - pub capture_lambda_payload_max_depth: Option, - /// @env `DD_COMPUTE_TRACE_STATS_ON_EXTENSION` - /// - /// If true, enable computation of trace stats on the extension side. - /// If false, trace stats will be computed on the backend side. - /// Default is `false`. - #[serde(deserialize_with = "deserialize_optional_bool_from_anything")] - pub compute_trace_stats_on_extension: Option, - /// @env `DD_SPAN_DEDUP_TIMEOUT` - /// - /// The timeout for the span deduplication service to check if a span key exists, in seconds. - /// For now, this is a temporary field added to debug the failure of `check_and_add()` in span dedup service. - /// Do not use this field extensively in production. - #[serde(deserialize_with = "deserialize_optional_duration_from_seconds_ignore_zero")] - pub span_dedup_timeout: Option, - /// @env `DD_API_KEY_SECRET_RELOAD_INTERVAL` - /// - /// The interval at which the Datadog API key is reloaded, in seconds. - /// If None, the API key will not be reloaded. - /// Default is `None`. - #[serde(deserialize_with = "deserialize_optional_duration_from_seconds_ignore_zero")] - pub api_key_secret_reload_interval: Option, - /// @env `DD_SERVERLESS_APPSEC_ENABLED` - /// - /// Enable Application and API Protection (AAP), previously known as AppSec/ASM, for AWS Lambda. - /// Default is `false`. - /// - #[serde(deserialize_with = "deserialize_optional_bool_from_anything")] - pub serverless_appsec_enabled: Option, - /// @env `DD_APPSEC_RULES` - /// - /// The path to a user-configured App & API Protection ruleset (in JSON format). - #[serde(deserialize_with = "deserialize_optional_string")] - pub appsec_rules: Option, - /// @env `DD_APPSEC_WAF_TIMEOUT` - /// - /// The timeout for the WAF to process a request, in microseconds. - #[serde(deserialize_with = "deserialize_optional_duration_from_microseconds")] - pub appsec_waf_timeout: Option, - /// @env `DD_API_SECURITY_ENABLED` - /// - /// Enable API Security for AWS Lambda. - #[serde(deserialize_with = "deserialize_optional_bool_from_anything")] - pub api_security_enabled: Option, - /// @env `DD_API_SECURITY_SAMPLE_DELAY` - /// - /// The delay between two samples of the API Security schema collection, in seconds. - #[serde(deserialize_with = "deserialize_optional_duration_from_seconds")] - pub api_security_sample_delay: Option, } #[allow(clippy::too_many_lines)] -fn merge_config(config: &mut Config, env_config: &EnvConfig) { +fn merge_config(config: &mut Config, env_config: &EnvConfig) { // Basic fields merge_string!(config, env_config, site); merge_string!(config, env_config, api_key); @@ -654,44 +541,19 @@ fn merge_config(config: &mut Config, env_config: &EnvConfig) { otlp_config_traces_probabilistic_sampler_sampling_percentage ); merge_option_to_value!(config, env_config, otlp_config_logs_enabled); - - // AWS Lambda - merge_string!(config, env_config, api_key_secret_arn); - merge_string!(config, env_config, kms_api_key); - merge_string!(config, env_config, api_key_ssm_arn); - merge_option_to_value!(config, env_config, serverless_logs_enabled); - - // Handle serverless_logs_enabled with OR logic: if either DD_LOGS_ENABLED or DD_SERVERLESS_LOGS_ENABLED is true, enable logs - if env_config.serverless_logs_enabled.is_some() || env_config.logs_enabled.is_some() { - config.serverless_logs_enabled = env_config.serverless_logs_enabled.unwrap_or(false) - || env_config.logs_enabled.unwrap_or(false); - } - - merge_option_to_value!(config, env_config, serverless_flush_strategy); - merge_option_to_value!(config, env_config, enhanced_metrics); - merge_option_to_value!(config, env_config, lambda_proc_enhanced_metrics); - merge_option_to_value!(config, env_config, capture_lambda_payload); - merge_option_to_value!(config, env_config, capture_lambda_payload_max_depth); - merge_option_to_value!(config, env_config, compute_trace_stats_on_extension); - merge_option!(config, env_config, span_dedup_timeout); - merge_option!(config, env_config, api_key_secret_reload_interval); - merge_option_to_value!(config, env_config, serverless_appsec_enabled); - merge_option!(config, env_config, appsec_rules); - merge_option_to_value!(config, env_config, appsec_waf_timeout); - merge_option_to_value!(config, env_config, api_security_enabled); - merge_option_to_value!(config, env_config, api_security_sample_delay); } #[derive(Debug, PartialEq, Clone, Copy)] #[allow(clippy::module_name_repetitions)] pub struct EnvConfigSource; -impl ConfigSource for EnvConfigSource { - fn load(&self, config: &mut Config) -> Result<(), ConfigError> { +impl ConfigSource for EnvConfigSource { + fn load(&self, config: &mut Config) -> Result<(), ConfigError> { let figment = Figment::new() .merge(Env::prefixed("DATADOG_")) .merge(Env::prefixed("DD_")); + // Extract core config fields match figment.extract::() { Ok(env_config) => merge_config(config, &env_config), Err(e) => { @@ -701,19 +563,27 @@ impl ConfigSource for EnvConfigSource { } } + // Extract extension fields via dual extraction + match figment.extract::() { + Ok(ext_source) => config.ext.merge_from(&ext_source), + Err(e) => { + tracing::warn!( + "Failed to parse extension config from environment variables: {e}, using default extension config." + ); + } + } + Ok(()) } } #[cfg_attr(coverage_nightly, coverage(off))] // Test modules skew coverage metrics #[cfg(test)] +#[allow(clippy::result_large_err)] mod tests { - use std::time::Duration; - use super::*; use crate::{ Config, TracePropagationStyle, - flush_strategy::{FlushStrategy, PeriodicStrategy}, log_level::LogLevel, processing_rule::{Kind, ProcessingRule}, }; @@ -727,6 +597,7 @@ mod tests { /// corresponding entry in the arrays below. #[test] #[allow(clippy::too_many_lines)] + #[allow(clippy::field_reassign_with_default)] fn test_all_env_fields_wrong_type_fallback_to_default() { // Non-string fields → invalid values that exercise graceful fallback. let invalid_non_string_env_vars: &[(&str, &str)] = &[ @@ -736,7 +607,6 @@ mod tests { ("DD_LOGS_CONFIG_COMPRESSION_LEVEL", "not_a_number"), ("DD_APM_CONFIG_COMPRESSION_LEVEL", "not_a_number"), ("DD_METRICS_CONFIG_COMPRESSION_LEVEL", "not_a_number"), - ("DD_CAPTURE_LAMBDA_PAYLOAD_MAX_DEPTH", "not_a_number"), ("DD_DOGSTATSD_SO_RCVBUF", "not_a_number"), ("DD_DOGSTATSD_BUFFER_SIZE", "not_a_number"), ("DD_DOGSTATSD_QUEUE_SIZE", "not_a_number"), @@ -763,12 +633,6 @@ mod tests { ("DD_TRACE_PROPAGATION_EXTRACT_FIRST", "not_a_bool"), ("DD_TRACE_PROPAGATION_HTTP_BAGGAGE_ENABLED", "not_a_bool"), ("DD_TRACE_AWS_SERVICE_REPRESENTATION_ENABLED", "not_a_bool"), - ("DD_ENHANCED_METRICS", "not_a_bool"), - ("DD_LAMBDA_PROC_ENHANCED_METRICS", "not_a_bool"), - ("DD_CAPTURE_LAMBDA_PAYLOAD", "not_a_bool"), - ("DD_COMPUTE_TRACE_STATS_ON_EXTENSION", "not_a_bool"), - ("DD_SERVERLESS_APPSEC_ENABLED", "not_a_bool"), - ("DD_API_SECURITY_ENABLED", "not_a_bool"), ("DD_OTLP_CONFIG_TRACES_ENABLED", "not_a_bool"), ( "DD_OTLP_CONFIG_TRACES_SPAN_NAME_AS_RESOURCE_NAME", @@ -797,16 +661,8 @@ mod tests { "DD_OBSERVABILITY_PIPELINES_WORKER_LOGS_ENABLED", "not_a_bool", ), - ("DD_SERVERLESS_LOGS_ENABLED", "not_a_bool"), - ("DD_LOGS_ENABLED", "not_a_bool"), // Enum ("DD_LOG_LEVEL", "invalid_level_999"), - ("DD_SERVERLESS_FLUSH_STRATEGY", "[[[invalid"), - // Duration - ("DD_SPAN_DEDUP_TIMEOUT", "not_a_number"), - ("DD_API_KEY_SECRET_RELOAD_INTERVAL", "not_a_number"), - ("DD_APPSEC_WAF_TIMEOUT", "not_a_number"), - ("DD_API_SECURITY_SAMPLE_DELAY", "not_a_number"), // JSON ("DD_ADDITIONAL_ENDPOINTS", "not_json{{"), ("DD_APM_ADDITIONAL_ENDPOINTS", "not_json{{"), @@ -870,16 +726,6 @@ mod tests { "keep", ), ("DD_OTLP_CONFIG_METRICS_SUMMARIES_MODE", "noquantiles"), - ( - "DD_API_KEY_SECRET_ARN", - "arn:aws:secretsmanager:us-east-1:123:secret:key", - ), - ("DD_KMS_API_KEY", "kms-encrypted-key"), - ( - "DD_API_KEY_SSM_ARN", - "arn:aws:ssm:us-east-1:123:parameter/key", - ), - ("DD_APPSEC_RULES", "/opt/custom-rules.json"), ]; // Programmatic guard: count `pub ` fields in the EnvConfig struct from @@ -912,7 +758,7 @@ mod tests { jail.set_env(key, value); } - let mut config = Config::default(); + let mut config: Config = Config::default(); // This MUST succeed — no single field should crash the whole config EnvConfigSource .load(&mut config) @@ -920,7 +766,7 @@ mod tests { // Build expected: string fields have their non-default values, // all non-string fields stay at defaults. - let mut expected = Config::default(); + let mut expected: Config = Config::default(); // String fields (merge_string! → Config String) expected.site = "custom-site.example.com".to_string(); expected.api_key = "test-api-key-12345".to_string(); @@ -930,10 +776,6 @@ mod tests { expected.observability_pipelines_worker_logs_url = "https://opw.example.com".to_string(); expected.apm_dd_url = "https://custom-apm.example.com".to_string(); - expected.api_key_secret_arn = - "arn:aws:secretsmanager:us-east-1:123:secret:key".to_string(); - expected.kms_api_key = "kms-encrypted-key".to_string(); - expected.api_key_ssm_arn = "arn:aws:ssm:us-east-1:123:parameter/key".to_string(); // Option fields (merge_option! → Config Option) expected.proxy_https = Some("https://proxy.example.com".to_string()); expected.http_protocol = Some("http1".to_string()); @@ -954,7 +796,6 @@ mod tests { expected.otlp_config_metrics_sums_initial_cumulativ_monotonic_value = Some("keep".to_string()); expected.otlp_config_metrics_summaries_mode = Some("noquantiles".to_string()); - expected.appsec_rules = Some("/opt/custom-rules.json".to_string()); assert_eq!(config, expected); Ok(()) @@ -1104,28 +945,7 @@ mod tests { jail.set_env("DD_DOGSTATSD_BUFFER_SIZE", "65507"); jail.set_env("DD_DOGSTATSD_QUEUE_SIZE", "2048"); - // AWS Lambda - jail.set_env( - "DD_API_KEY_SECRET_ARN", - "arn:aws:secretsmanager:region:account:secret:datadog-api-key", - ); - jail.set_env("DD_KMS_API_KEY", "test-kms-key"); - jail.set_env("DD_SERVERLESS_LOGS_ENABLED", "false"); - jail.set_env("DD_SERVERLESS_FLUSH_STRATEGY", "periodically,60000"); - jail.set_env("DD_ENHANCED_METRICS", "false"); - jail.set_env("DD_LAMBDA_PROC_ENHANCED_METRICS", "false"); - jail.set_env("DD_CAPTURE_LAMBDA_PAYLOAD", "true"); - jail.set_env("DD_CAPTURE_LAMBDA_PAYLOAD_MAX_DEPTH", "5"); - jail.set_env("DD_COMPUTE_TRACE_STATS_ON_EXTENSION", "true"); - jail.set_env("DD_SPAN_DEDUP_TIMEOUT", "5"); - jail.set_env("DD_API_KEY_SECRET_RELOAD_INTERVAL", "10"); - jail.set_env("DD_SERVERLESS_APPSEC_ENABLED", "true"); - jail.set_env("DD_APPSEC_RULES", "/path/to/rules.json"); - jail.set_env("DD_APPSEC_WAF_TIMEOUT", "1000000"); // Microseconds - jail.set_env("DD_API_SECURITY_ENABLED", "0"); // Seconds - jail.set_env("DD_API_SECURITY_SAMPLE_DELAY", "60"); // Seconds - - let mut config = Config::default(); + let mut config: Config = Config::default(); let env_config_source = EnvConfigSource; env_config_source .load(&mut config) @@ -1262,26 +1082,7 @@ mod tests { dogstatsd_so_rcvbuf: Some(1_048_576), dogstatsd_buffer_size: Some(65507), dogstatsd_queue_size: Some(2048), - api_key_secret_arn: "arn:aws:secretsmanager:region:account:secret:datadog-api-key" - .to_string(), - kms_api_key: "test-kms-key".to_string(), - api_key_ssm_arn: String::default(), - serverless_logs_enabled: false, - serverless_flush_strategy: FlushStrategy::Periodically(PeriodicStrategy { - interval: 60000, - }), - enhanced_metrics: false, - lambda_proc_enhanced_metrics: false, - capture_lambda_payload: true, - capture_lambda_payload_max_depth: 5, - compute_trace_stats_on_extension: true, - span_dedup_timeout: Some(Duration::from_secs(5)), - api_key_secret_reload_interval: Some(Duration::from_secs(10)), - serverless_appsec_enabled: true, - appsec_rules: Some("/path/to/rules.json".to_string()), - appsec_waf_timeout: Duration::from_secs(1), - api_security_enabled: false, - api_security_sample_delay: Duration::from_secs(60), + ext: crate::NoExtension, }; assert_eq!(config, expected_config); @@ -1290,165 +1091,6 @@ mod tests { }); } - #[test] - fn test_dd_logs_enabled_true() { - figment::Jail::expect_with(|jail| { - jail.clear_env(); - jail.set_env("DD_LOGS_ENABLED", "true"); - - let mut config = Config::default(); - let env_config_source = EnvConfigSource; - env_config_source - .load(&mut config) - .expect("Failed to load config"); - - assert!(config.serverless_logs_enabled); - Ok(()) - }); - } - - #[test] - fn test_dd_logs_enabled_false() { - figment::Jail::expect_with(|jail| { - jail.clear_env(); - jail.set_env("DD_LOGS_ENABLED", "false"); - - let mut config = Config::default(); - let env_config_source = EnvConfigSource; - env_config_source - .load(&mut config) - .expect("Failed to load config"); - - assert!(!config.serverless_logs_enabled); - Ok(()) - }); - } - - #[test] - fn test_dd_serverless_logs_enabled_true() { - figment::Jail::expect_with(|jail| { - jail.clear_env(); - jail.set_env("DD_SERVERLESS_LOGS_ENABLED", "true"); - - let mut config = Config::default(); - let env_config_source = EnvConfigSource; - env_config_source - .load(&mut config) - .expect("Failed to load config"); - - assert!(config.serverless_logs_enabled); - Ok(()) - }); - } - - #[test] - fn test_dd_serverless_logs_enabled_false() { - figment::Jail::expect_with(|jail| { - jail.clear_env(); - jail.set_env("DD_SERVERLESS_LOGS_ENABLED", "false"); - - let mut config = Config::default(); - let env_config_source = EnvConfigSource; - env_config_source - .load(&mut config) - .expect("Failed to load config"); - - assert!(!config.serverless_logs_enabled); - Ok(()) - }); - } - - #[test] - fn test_both_logs_enabled_true() { - figment::Jail::expect_with(|jail| { - jail.clear_env(); - jail.set_env("DD_LOGS_ENABLED", "true"); - jail.set_env("DD_SERVERLESS_LOGS_ENABLED", "true"); - - let mut config = Config::default(); - let env_config_source = EnvConfigSource; - env_config_source - .load(&mut config) - .expect("Failed to load config"); - - assert!(config.serverless_logs_enabled); - Ok(()) - }); - } - - #[test] - fn test_both_logs_enabled_false() { - figment::Jail::expect_with(|jail| { - jail.clear_env(); - jail.set_env("DD_LOGS_ENABLED", "false"); - jail.set_env("DD_SERVERLESS_LOGS_ENABLED", "false"); - - let mut config = Config::default(); - let env_config_source = EnvConfigSource; - env_config_source - .load(&mut config) - .expect("Failed to load config"); - - assert!(!config.serverless_logs_enabled); - Ok(()) - }); - } - - #[test] - fn test_logs_enabled_true_serverless_logs_enabled_false() { - figment::Jail::expect_with(|jail| { - jail.clear_env(); - jail.set_env("DD_LOGS_ENABLED", "true"); - jail.set_env("DD_SERVERLESS_LOGS_ENABLED", "false"); - - let mut config = Config::default(); - let env_config_source = EnvConfigSource; - env_config_source - .load(&mut config) - .expect("Failed to load config"); - - // OR logic: if either is true, logs are enabled - assert!(config.serverless_logs_enabled); - Ok(()) - }); - } - - #[test] - fn test_logs_enabled_false_serverless_logs_enabled_true() { - figment::Jail::expect_with(|jail| { - jail.clear_env(); - jail.set_env("DD_LOGS_ENABLED", "false"); - jail.set_env("DD_SERVERLESS_LOGS_ENABLED", "true"); - - let mut config = Config::default(); - let env_config_source = EnvConfigSource; - env_config_source - .load(&mut config) - .expect("Failed to load config"); - - // OR logic: if either is true, logs are enabled - assert!(config.serverless_logs_enabled); - Ok(()) - }); - } - - #[test] - fn test_neither_logs_enabled_set_uses_default() { - figment::Jail::expect_with(|jail| { - jail.clear_env(); - - let mut config = Config::default(); - let env_config_source = EnvConfigSource; - env_config_source - .load(&mut config) - .expect("Failed to load config"); - - // Default value is true - assert!(config.serverless_logs_enabled); - Ok(()) - }); - } - #[test] fn test_dogstatsd_config_from_env() { figment::Jail::expect_with(|jail| { @@ -1457,7 +1099,7 @@ mod tests { jail.set_env("DD_DOGSTATSD_BUFFER_SIZE", "65507"); jail.set_env("DD_DOGSTATSD_QUEUE_SIZE", "2048"); - let mut config = Config::default(); + let mut config: Config = Config::default(); let env_config_source = EnvConfigSource; env_config_source .load(&mut config) @@ -1475,7 +1117,7 @@ mod tests { figment::Jail::expect_with(|jail| { jail.clear_env(); - let mut config = Config::default(); + let mut config: Config = Config::default(); let env_config_source = EnvConfigSource; env_config_source .load(&mut config) diff --git a/crates/datadog-agent-config/src/sources/mod.rs b/crates/datadog-agent-config/src/sources/mod.rs new file mode 100644 index 0000000..dc4d398 --- /dev/null +++ b/crates/datadog-agent-config/src/sources/mod.rs @@ -0,0 +1,2 @@ +pub mod env; +pub mod yaml; diff --git a/crates/datadog-agent-config/yaml.rs b/crates/datadog-agent-config/src/sources/yaml.rs similarity index 85% rename from crates/datadog-agent-config/yaml.rs rename to crates/datadog-agent-config/src/sources/yaml.rs index 06b7851..933a9fd 100644 --- a/crates/datadog-agent-config/yaml.rs +++ b/crates/datadog-agent-config/src/sources/yaml.rs @@ -1,15 +1,12 @@ -use std::time::Duration; use std::{collections::HashMap, path::PathBuf}; use crate::{ - Config, ConfigError, ConfigSource, ProcessingRule, TracePropagationStyle, + Config, ConfigError, ConfigExtension, ConfigSource, ProcessingRule, TracePropagationStyle, additional_endpoints::deserialize_additional_endpoints, deserialize_apm_replace_rules, deserialize_key_value_pair_array_to_hashmap, deserialize_option_lossless, - deserialize_optional_bool_from_anything, deserialize_optional_duration_from_microseconds, - deserialize_optional_duration_from_seconds, - deserialize_optional_duration_from_seconds_ignore_zero, deserialize_optional_string, + deserialize_optional_bool_from_anything, deserialize_optional_string, deserialize_processing_rules, deserialize_string_or_int, deserialize_trace_propagation_style, - deserialize_with_default, flush_strategy::FlushStrategy, log_level::LogLevel, + deserialize_with_default, log_level::LogLevel, logs_additional_endpoints::LogsAdditionalEndpoint, merge_hashmap, merge_option, merge_option_to_value, merge_string, merge_vec, service_mapping::deserialize_service_mapping, }; @@ -108,40 +105,6 @@ pub struct YamlConfig { // OTLP #[serde(deserialize_with = "deserialize_with_default")] pub otlp_config: Option, - - // AWS Lambda - #[serde(deserialize_with = "deserialize_optional_string")] - pub api_key_secret_arn: Option, - #[serde(deserialize_with = "deserialize_optional_string")] - pub kms_api_key: Option, - #[serde(deserialize_with = "deserialize_optional_bool_from_anything")] - pub serverless_logs_enabled: Option, - #[serde(deserialize_with = "deserialize_optional_bool_from_anything")] - pub logs_enabled: Option, - #[serde(deserialize_with = "deserialize_with_default")] - pub serverless_flush_strategy: Option, - #[serde(deserialize_with = "deserialize_optional_bool_from_anything")] - pub enhanced_metrics: Option, - #[serde(deserialize_with = "deserialize_optional_bool_from_anything")] - pub lambda_proc_enhanced_metrics: Option, - #[serde(deserialize_with = "deserialize_optional_bool_from_anything")] - pub capture_lambda_payload: Option, - #[serde(deserialize_with = "deserialize_option_lossless")] - pub capture_lambda_payload_max_depth: Option, - #[serde(deserialize_with = "deserialize_optional_bool_from_anything")] - pub compute_trace_stats_on_extension: Option, - #[serde(deserialize_with = "deserialize_optional_duration_from_seconds_ignore_zero")] - pub api_key_secret_reload_interval: Option, - #[serde(deserialize_with = "deserialize_optional_bool_from_anything")] - pub serverless_appsec_enabled: Option, - #[serde(deserialize_with = "deserialize_optional_string")] - pub appsec_rules: Option, - #[serde(deserialize_with = "deserialize_optional_duration_from_microseconds")] - pub appsec_waf_timeout: Option, - #[serde(deserialize_with = "deserialize_optional_bool_from_anything")] - pub api_security_enabled: Option, - #[serde(deserialize_with = "deserialize_optional_duration_from_seconds")] - pub api_security_sample_delay: Option, } /// Proxy Config @@ -443,7 +406,7 @@ impl OtlpConfig { } #[allow(clippy::too_many_lines)] -fn merge_config(config: &mut Config, yaml_config: &YamlConfig) { +fn merge_config(config: &mut Config, yaml_config: &YamlConfig) { // Basic fields merge_string!(config, yaml_config, site); merge_string!(config, yaml_config, api_key); @@ -720,29 +683,6 @@ fn merge_config(config: &mut Config, yaml_config: &YamlConfig) { merge_option_to_value!(config, otlp_config_logs_enabled, logs, enabled); } } - - // AWS Lambda - merge_string!(config, yaml_config, api_key_secret_arn); - merge_string!(config, yaml_config, kms_api_key); - - // Handle serverless_logs_enabled with OR logic: if either logs_enabled or serverless_logs_enabled is true, enable logs - if yaml_config.serverless_logs_enabled.is_some() || yaml_config.logs_enabled.is_some() { - config.serverless_logs_enabled = yaml_config.serverless_logs_enabled.unwrap_or(false) - || yaml_config.logs_enabled.unwrap_or(false); - } - - merge_option_to_value!(config, yaml_config, serverless_flush_strategy); - merge_option_to_value!(config, yaml_config, enhanced_metrics); - merge_option_to_value!(config, yaml_config, lambda_proc_enhanced_metrics); - merge_option_to_value!(config, yaml_config, capture_lambda_payload); - merge_option_to_value!(config, yaml_config, capture_lambda_payload_max_depth); - merge_option_to_value!(config, yaml_config, compute_trace_stats_on_extension); - merge_option!(config, yaml_config, api_key_secret_reload_interval); - merge_option_to_value!(config, yaml_config, serverless_appsec_enabled); - merge_option!(config, yaml_config, appsec_rules); - merge_option_to_value!(config, yaml_config, appsec_waf_timeout); - merge_option_to_value!(config, yaml_config, api_security_enabled); - merge_option_to_value!(config, yaml_config, api_security_sample_delay); } #[derive(Debug, PartialEq, Clone)] @@ -751,8 +691,8 @@ pub struct YamlConfigSource { pub path: PathBuf, } -impl ConfigSource for YamlConfigSource { - fn load(&self, config: &mut Config) -> Result<(), ConfigError> { +impl ConfigSource for YamlConfigSource { + fn load(&self, config: &mut Config) -> Result<(), ConfigError> { let figment = Figment::new().merge(Yaml::file(self.path.clone())); match figment.extract::() { @@ -764,17 +704,27 @@ impl ConfigSource for YamlConfigSource { } } + // Extract extension fields via dual extraction + match figment.extract::() { + Ok(ext_source) => config.ext.merge_from(&ext_source), + Err(e) => { + tracing::warn!( + "Failed to parse extension config from yaml file: {e}, using default extension config." + ); + } + } + Ok(()) } } #[cfg_attr(coverage_nightly, coverage(off))] // Test modules skew coverage metrics #[cfg(test)] +#[allow(clippy::result_large_err)] mod tests { use std::path::Path; - use std::time::Duration; - use crate::{flush_strategy::PeriodicStrategy, log_level::LogLevel, processing_rule::Kind}; + use crate::{log_level::LogLevel, processing_rule::Kind}; use super::*; @@ -784,6 +734,7 @@ mod tests { /// When adding a new field to YamlConfig or any nested struct, add an entry /// here with the wrong type to ensure graceful deserialization is in place. #[test] + #[allow(clippy::field_reassign_with_default)] fn test_all_yaml_fields_wrong_type_fallback_to_default() { figment::Jail::expect_with(|jail| { jail.clear_env(); @@ -890,28 +841,10 @@ otlp_config: mode: "noquantiles" logs: enabled: [1, 2, 3] - -# AWS Lambda -api_key_secret_arn: "arn:aws:secretsmanager:us-east-1:123:secret:key" -kms_api_key: "kms-encrypted-key" -serverless_logs_enabled: [1, 2, 3] -logs_enabled: [1, 2, 3] -serverless_flush_strategy: [1, 2, 3] -enhanced_metrics: [1, 2, 3] -lambda_proc_enhanced_metrics: [1, 2, 3] -capture_lambda_payload: [1, 2, 3] -capture_lambda_payload_max_depth: [1, 2, 3] -compute_trace_stats_on_extension: [1, 2, 3] -api_key_secret_reload_interval: [1, 2, 3] -serverless_appsec_enabled: [1, 2, 3] -appsec_rules: "/opt/custom-rules.json" -appsec_waf_timeout: [1, 2, 3] -api_security_enabled: [1, 2, 3] -api_security_sample_delay: [1, 2, 3] "#, )?; - let mut config = Config::default(); + let mut config: Config = Config::default(); let source = YamlConfigSource { path: PathBuf::from("datadog.yaml"), }; @@ -922,15 +855,12 @@ api_security_sample_delay: [1, 2, 3] // Build expected: string fields have their non-default values, // all non-string fields stay at defaults. - let mut expected = Config::default(); + let mut expected: Config = Config::default(); expected.site = "custom-site.example.com".to_string(); expected.api_key = "test-api-key-12345".to_string(); expected.dd_url = "https://custom-metrics.example.com".to_string(); expected.logs_config_logs_dd_url = "https://custom-logs.example.com".to_string(); expected.apm_dd_url = "https://custom-apm.example.com".to_string(); - expected.api_key_secret_arn = - "arn:aws:secretsmanager:us-east-1:123:secret:key".to_string(); - expected.kms_api_key = "kms-encrypted-key".to_string(); // Option fields expected.proxy_https = Some("https://proxy.example.com".to_string()); expected.http_protocol = Some("http1".to_string()); @@ -950,7 +880,6 @@ api_security_sample_delay: [1, 2, 3] expected.otlp_config_metrics_sums_initial_cumulativ_monotonic_value = Some("keep".to_string()); expected.otlp_config_metrics_summaries_mode = Some("noquantiles".to_string()); - expected.appsec_rules = Some("/opt/custom-rules.json".to_string()); assert_eq!(config, expected); Ok(()) @@ -1081,27 +1010,10 @@ otlp_config: mode: "quantiles" logs: enabled: true - -# AWS Lambda -api_key_secret_arn: "arn:aws:secretsmanager:region:account:secret:datadog-api-key" -kms_api_key: "test-kms-key" -serverless_logs_enabled: false -serverless_flush_strategy: "periodically,60000" -enhanced_metrics: false -lambda_proc_enhanced_metrics: false -capture_lambda_payload: true -capture_lambda_payload_max_depth: 5 -compute_trace_stats_on_extension: true -api_key_secret_reload_interval: 0 -serverless_appsec_enabled: true -appsec_rules: "/path/to/rules.json" -appsec_waf_timeout: 1000000 # Microseconds -api_security_enabled: false -api_security_sample_delay: 60 # Seconds "#, )?; - let mut config = Config::default(); + let mut config: Config = Config::default(); let yaml_config_source = YamlConfigSource { path: Path::new("datadog.yaml").to_path_buf(), }; @@ -1215,28 +1127,6 @@ api_security_sample_delay: 60 # Seconds otlp_config_metrics_summaries_mode: Some("quantiles".to_string()), otlp_config_traces_probabilistic_sampler_sampling_percentage: Some(50), otlp_config_logs_enabled: true, - api_key_secret_arn: "arn:aws:secretsmanager:region:account:secret:datadog-api-key" - .to_string(), - kms_api_key: "test-kms-key".to_string(), - api_key_ssm_arn: String::default(), - serverless_logs_enabled: false, - serverless_flush_strategy: FlushStrategy::Periodically(PeriodicStrategy { - interval: 60000, - }), - enhanced_metrics: false, - lambda_proc_enhanced_metrics: false, - capture_lambda_payload: true, - capture_lambda_payload_max_depth: 5, - compute_trace_stats_on_extension: true, - span_dedup_timeout: None, - api_key_secret_reload_interval: None, - - serverless_appsec_enabled: true, - appsec_rules: Some("/path/to/rules.json".to_string()), - appsec_waf_timeout: Duration::from_secs(1), - api_security_enabled: false, - api_security_sample_delay: Duration::from_secs(60), - apm_filter_tags_require: None, apm_filter_tags_reject: None, apm_filter_tags_regex_require: None, @@ -1245,6 +1135,7 @@ api_security_sample_delay: 60 # Seconds dogstatsd_so_rcvbuf: Some(1_048_576), dogstatsd_buffer_size: Some(65507), dogstatsd_queue_size: Some(2048), + ext: crate::NoExtension, }; // Assert that @@ -1266,7 +1157,7 @@ dogstatsd_buffer_size: 16384 dogstatsd_queue_size: 512 ", )?; - let mut config = Config::default(); + let mut config: Config = Config::default(); let yaml_config_source = YamlConfigSource { path: Path::new("datadog.yaml").to_path_buf(), }; @@ -1286,7 +1177,7 @@ dogstatsd_queue_size: 512 figment::Jail::expect_with(|jail| { jail.clear_env(); jail.create_file("datadog.yaml", "")?; - let mut config = Config::default(); + let mut config: Config = Config::default(); let yaml_config_source = YamlConfigSource { path: Path::new("datadog.yaml").to_path_buf(), }; diff --git a/crates/datadog-fips/Cargo.toml b/crates/datadog-fips/Cargo.toml index 9758f41..e09a358 100644 --- a/crates/datadog-fips/Cargo.toml +++ b/crates/datadog-fips/Cargo.toml @@ -9,7 +9,7 @@ repository.workspace = true [dependencies] reqwest = { version = "0.12.4", features = ["json", "http2"], default-features = false } rustls = { version = "0.23.18", default-features = false, features = ["fips"], optional = true } -rustls-native-certs = { version = "0.8.1", optional = true } +rustls-native-certs = { version = ">=0.8.1, <0.8.3", optional = true } tracing = { version = "0.1.40", default-features = false } [features] diff --git a/crates/datadog-logs-agent/Cargo.toml b/crates/datadog-logs-agent/Cargo.toml new file mode 100644 index 0000000..177d561 --- /dev/null +++ b/crates/datadog-logs-agent/Cargo.toml @@ -0,0 +1,35 @@ +# Copyright 2025-Present Datadog, Inc. https://www.datadoghq.com/ +# SPDX-License-Identifier: Apache-2.0 + +[package] +name = "datadog-logs-agent" +version = "0.1.0" +edition.workspace = true +license.workspace = true + +[lib] +bench = false + +[dependencies] +datadog-fips = { path = "../datadog-fips" } +reqwest = { version = "0.12.4", features = ["json", "http2"], default-features = false } +serde = { version = "1.0.197", default-features = false, features = ["derive"] } +serde_json = { version = "1.0.116", default-features = false, features = ["alloc"] } +thiserror = { version = "1.0.58", default-features = false } +hyper = { version = "1", features = ["http1", "server"] } +http-body-util = "0.1" +hyper-util = { version = "0.1", features = ["tokio"] } +futures = { version = "0.3", default-features = false, features = ["alloc"] } +tokio = { version = "1.37.0", default-features = false, features = ["sync", "net"] } +tracing = { version = "0.1.40", default-features = false } +zstd = { version = "0.13.3", default-features = false } + +[dev-dependencies] +http = "1" +mockito = { version = "1.5.0", default-features = false } +serde_json = { version = "1.0.116", default-features = false, features = ["alloc"] } +reqwest = { version = "0.12.4", features = ["json"], default-features = false } +tokio = { version = "1.37.0", default-features = false, features = ["macros", "rt-multi-thread", "net", "time"] } + +[features] +default = [] diff --git a/crates/datadog-logs-agent/examples/send_logs.rs b/crates/datadog-logs-agent/examples/send_logs.rs new file mode 100644 index 0000000..7d64df9 --- /dev/null +++ b/crates/datadog-logs-agent/examples/send_logs.rs @@ -0,0 +1,196 @@ +// Copyright 2023-Present Datadog, Inc. https://www.datadoghq.com/ +// SPDX-License-Identifier: Apache-2.0 + +//! Local test helper: inserts sample log entries and flushes them via the log agent pipeline. +//! +//! # Usage +//! +//! ## Flush to a local capture server (recommended for local dev) +//! +//! The easiest way — runs capture server and example together: +//! ./scripts/test-log-intake.sh +//! +//! Or manually in two terminals: +//! +//! In terminal 1 — start the capture server (handles POST, prints JSON): +//! python3 scripts/test-log-intake.sh # not available standalone +//! # Use the script above, or run: python3 -c "$(sed -n '/PYEOF/,/PYEOF/p' scripts/test-log-intake.sh)" +//! +//! In terminal 2 — run this example: +//! DD_OBSERVABILITY_PIPELINES_WORKER_LOGS_ENABLED=true \ +//! DD_OBSERVABILITY_PIPELINES_WORKER_LOGS_URL=http://localhost:9999/logs \ +//! DD_API_KEY=local-test-key \ +//! cargo run -p datadog-logs-agent --example send_logs +//! +//! NOTE: `python3 -m http.server` does NOT work — it rejects POST requests. +//! +//! ## Flush to a real Datadog endpoint +//! +//! DD_API_KEY= \ +//! DD_SITE=datadoghq.com \ +//! cargo run -p datadog-logs-agent --example send_logs +//! +//! ## Configuration via env vars +//! +//! | Variable | Default | +//! |--------------------------------------------------|--------------------| +//! | DD_API_KEY | (empty) | +//! | DD_SITE | datadoghq.com | +//! | DD_LOGS_CONFIG_USE_COMPRESSION | true | +//! | DD_LOGS_CONFIG_COMPRESSION_LEVEL | 3 | +//! | DD_OBSERVABILITY_PIPELINES_WORKER_LOGS_ENABLED | false | +//! | DD_OBSERVABILITY_PIPELINES_WORKER_LOGS_URL | (empty) | +//! | LOG_ENTRY_COUNT | 5 | + +use datadog_logs_agent::{ + AggregatorService, Destination, IntakeEntry, LogFlusher, LogFlusherConfig, +}; + +#[allow(clippy::disallowed_methods)] // plain reqwest::Client for local testing +#[tokio::main] +async fn main() { + let entry_count: usize = std::env::var("LOG_ENTRY_COUNT") + .ok() + .and_then(|v| v.parse().ok()) + .unwrap_or(5); + + let config = LogFlusherConfig::from_env(); + + // Print effective configuration + let (endpoint, compressed) = describe_config(&config); + println!("──────────────────────────────────────────"); + println!(" datadog-logs-agent local test"); + println!("──────────────────────────────────────────"); + println!(" endpoint : {endpoint}"); + println!(" api_key : {}", mask(&config.api_key)); + println!(" compressed : {compressed}"); + println!(" entries : {entry_count}"); + println!("──────────────────────────────────────────"); + + // Start aggregator service + let (service, handle) = AggregatorService::new(); + tokio::spawn(service.run()); + + // Insert sample entries representing different runtimes + let mut entries = Vec::with_capacity(entry_count); + + for i in 0..entry_count { + let entry = match i % 3 { + 0 => lambda_entry(i), + 1 => azure_entry(i), + _ => plain_entry(i), + }; + entries.push(entry); + } + + println!("\nInserting {entry_count} log entries..."); + handle.insert_batch(entries).expect("insert_batch failed"); + + // Build HTTP client + let client = reqwest::Client::builder() + .timeout(config.flush_timeout) + .build() + .expect("failed to build HTTP client"); + + // Flush + println!("Flushing to {endpoint}..."); + let flusher = LogFlusher::new(config, client, handle); + let failed = flusher.flush(vec![]).await; + + if failed.is_empty() { + println!("\n✓ Flush succeeded"); + } else { + eprintln!("\n✗ Flush failed — check endpoint and API key"); + std::process::exit(1); + } +} + +// ── Sample log entry builders ───────────────────────────────────────────────── + +fn lambda_entry(i: usize) -> IntakeEntry { + let mut attrs = serde_json::Map::new(); + attrs.insert( + "lambda".to_string(), + serde_json::json!({ + "arn": "arn:aws:lambda:us-east-1:123456789012:function:my-fn", + "request_id": format!("req-{i:04}") + }), + ); + IntakeEntry { + message: format!("[lambda] invocation #{i} completed"), + timestamp: now_ms(), + hostname: Some("arn:aws:lambda:us-east-1:123456789012:function:my-fn".to_string()), + service: Some("my-fn".to_string()), + ddsource: Some("lambda".to_string()), + ddtags: Some("env:local,runtime:lambda".to_string()), + status: Some("info".to_string()), + attributes: attrs, + } +} + +fn azure_entry(i: usize) -> IntakeEntry { + let mut attrs = serde_json::Map::new(); + attrs.insert( + "azure".to_string(), + serde_json::json!({ + "resource_id": "/subscriptions/sub-123/resourceGroups/rg/providers/Microsoft.Web/sites/my-fn", + "operation_name": "Microsoft.Web/sites/functions/run/action" + }), + ); + IntakeEntry { + message: format!("[azure] function triggered #{i}"), + timestamp: now_ms(), + hostname: Some("my-azure-fn".to_string()), + service: Some("payments".to_string()), + ddsource: Some("azure-functions".to_string()), + ddtags: Some("env:local,runtime:azure".to_string()), + status: Some("info".to_string()), + attributes: attrs, + } +} + +fn plain_entry(i: usize) -> IntakeEntry { + IntakeEntry { + message: format!("[generic] log message #{i}"), + timestamp: now_ms(), + hostname: Some("localhost".to_string()), + service: Some("test-service".to_string()), + ddsource: Some("rust".to_string()), + ddtags: Some("env:local".to_string()), + status: if i.is_multiple_of(5) { + Some("error".to_string()) + } else { + Some("info".to_string()) + }, + attributes: serde_json::Map::new(), + } +} + +// ── Helpers ─────────────────────────────────────────────────────────────────── + +fn now_ms() -> i64 { + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map(|d| d.as_millis() as i64) + .unwrap_or(0) +} + +fn describe_config(config: &LogFlusherConfig) -> (String, bool) { + match &config.mode { + Destination::Datadog => ( + format!("https://http-intake.logs.{}/api/v2/logs", config.site), + config.use_compression, + ), + Destination::ObservabilityPipelinesWorker { url } => (url.clone(), false), + } +} + +fn mask(s: &str) -> String { + if s.is_empty() { + return "(not set)".to_string(); + } + if s.len() <= 8 { + return "*".repeat(s.len()); + } + format!("{}…{}", &s[..4], &s[s.len() - 4..]) +} diff --git a/crates/datadog-logs-agent/src/aggregator/core.rs b/crates/datadog-logs-agent/src/aggregator/core.rs new file mode 100644 index 0000000..97a64f4 --- /dev/null +++ b/crates/datadog-logs-agent/src/aggregator/core.rs @@ -0,0 +1,270 @@ +// Copyright 2025-Present Datadog, Inc. https://www.datadoghq.com/ +// SPDX-License-Identifier: Apache-2.0 + +use std::collections::VecDeque; + +use crate::constants::{MAX_BATCH_ENTRIES, MAX_CONTENT_BYTES, MAX_LOG_BYTES}; +use crate::errors::AggregatorError; +use crate::intake_entry::IntakeEntry; + +/// In-memory log batch accumulator. +/// +/// Stores pre-serialized JSON strings in a FIFO queue. A batch is "full" +/// when it reaches `MAX_BATCH_ENTRIES` entries or `MAX_CONTENT_BYTES` of +/// uncompressed content. +pub struct Aggregator { + messages: VecDeque, + current_size_bytes: usize, +} + +impl Default for Aggregator { + fn default() -> Self { + Self::new() + } +} + +impl Aggregator { + /// Create a new, empty aggregator. + pub fn new() -> Self { + Self { + messages: VecDeque::new(), + current_size_bytes: 0, + } + } + + /// Insert a log entry into the batch. + /// + /// Returns `Ok(true)` if the batch is now full and ready to flush. + /// Returns `Err(AggregatorError::EntryTooLarge)` if the serialized + /// entry exceeds `MAX_LOG_BYTES` — the entry is dropped. + pub fn insert(&mut self, entry: &IntakeEntry) -> Result { + let serialized = serde_json::to_string(entry)?; + let len = serialized.len(); + + if len > MAX_LOG_BYTES { + return Err(AggregatorError::EntryTooLarge { + size: len, + max: MAX_LOG_BYTES, + }); + } + + self.messages.push_back(serialized); + self.current_size_bytes += len; + + Ok(self.is_full()) + } + + /// Returns `true` if the batch has reached its entry count or byte limit. + /// + /// The byte check accounts for JSON framing: `[` + `]` (2 bytes) plus one + /// comma per entry after the first (`N - 1` bytes). + pub fn is_full(&self) -> bool { + let n = self.messages.len(); + // framing: 2 bytes for `[`/`]` + (n - 1) commas + let framing = if n == 0 { 0 } else { 2 + (n - 1) }; + n >= MAX_BATCH_ENTRIES || self.current_size_bytes + framing >= MAX_CONTENT_BYTES + } + + /// Returns `true` if no log entries are buffered. + pub fn is_empty(&self) -> bool { + self.messages.is_empty() + } + + /// Returns the number of log entries currently buffered. + pub fn len(&self) -> usize { + self.messages.len() + } + + /// Drain up to `MAX_BATCH_ENTRIES` entries and return them as a JSON + /// array (`[entry1,entry2,...]`) encoded as UTF-8 bytes. + /// + /// Returns `None` if the aggregator is empty. + pub fn get_batch(&mut self) -> Option> { + if self.messages.is_empty() { + return None; + } + + let mut output = Vec::new(); + output.push(b'['); + let mut bytes_in_batch: usize = 0; + let mut count: usize = 0; + + loop { + if count >= MAX_BATCH_ENTRIES { + break; + } + + let msg_len = match self.messages.front() { + Some(m) => m.len(), + None => break, + }; + + // Account for the comma separator and the 2-byte `[`/`]` framing. + // Total wire size = bytes_in_batch + (separator) + msg_len + 2 (for `[` and `]`) + let separator = if count == 0 { 0 } else { 1 }; + if bytes_in_batch + separator + msg_len + 2 > MAX_CONTENT_BYTES { + break; + } + + // Safe: we just confirmed front() is Some and we hold &mut self + let msg = match self.messages.pop_front() { + Some(m) => m, + None => break, + }; + + if count > 0 { + output.push(b','); + bytes_in_batch += 1; + } + + self.current_size_bytes = self.current_size_bytes.saturating_sub(msg.len()); + bytes_in_batch += msg.len(); + count += 1; + output.extend_from_slice(msg.as_bytes()); + } + + output.push(b']'); + Some(output) + } + + /// Drain all entries and return them as a `Vec` of JSON array batches. + /// May return multiple batches if the queue exceeds a single batch limit. + pub fn get_all_batches(&mut self) -> Vec> { + let mut batches = Vec::new(); + while let Some(batch) = self.get_batch() { + batches.push(batch); + } + batches + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::intake_entry::IntakeEntry; + + fn make_entry(msg: &str) -> IntakeEntry { + IntakeEntry::from_message(msg, 1_700_000_000_000) + } + + #[test] + fn test_new_aggregator_is_empty() { + let agg = Aggregator::new(); + assert!(agg.is_empty()); + assert_eq!(agg.len(), 0); + } + + #[test] + fn test_insert_single_entry() { + let mut agg = Aggregator::new(); + let full = agg.insert(&make_entry("hello")).expect("insert failed"); + assert!(!full, "single entry should not fill the batch"); + assert_eq!(agg.len(), 1); + assert!(!agg.is_empty()); + } + + #[test] + fn test_get_batch_returns_valid_json_array() { + let mut agg = Aggregator::new(); + agg.insert(&make_entry("line 1")).expect("insert"); + agg.insert(&make_entry("line 2")).expect("insert"); + + let batch = agg.get_batch().expect("should have a batch"); + let parsed: serde_json::Value = serde_json::from_slice(&batch).expect("valid JSON"); + + assert!(parsed.is_array()); + assert_eq!(parsed.as_array().unwrap().len(), 2); + assert_eq!(parsed[0]["message"], "line 1"); + assert_eq!(parsed[1]["message"], "line 2"); + } + + #[test] + fn test_get_batch_drains_aggregator() { + let mut agg = Aggregator::new(); + agg.insert(&make_entry("log")).expect("insert"); + + let _ = agg.get_batch(); + assert!(agg.is_empty(), "aggregator should be empty after get_batch"); + assert!( + agg.get_batch().is_none(), + "second get_batch should return None" + ); + } + + #[test] + fn test_entry_too_large_returns_error() { + let mut agg = Aggregator::new(); + // Construct an entry whose JSON serialization exceeds MAX_LOG_BYTES + let big_message = "x".repeat(crate::constants::MAX_LOG_BYTES + 1); + let entry = make_entry(&big_message); + let result = agg.insert(&entry); + assert!(result.is_err(), "oversized entry should be rejected"); + assert!( + agg.is_empty(), + "aggregator should stay empty after rejection" + ); + } + + #[test] + fn test_insert_returns_true_when_batch_full_by_count() { + use crate::constants::MAX_BATCH_ENTRIES; + let mut agg = Aggregator::new(); + let entry = make_entry("x"); + + for _ in 0..(MAX_BATCH_ENTRIES - 1) { + let full = agg.insert(&entry).expect("insert"); + assert!(!full, "should not be full until last entry"); + } + let full = agg.insert(&entry).expect("insert"); + assert!(full, "should be full after MAX_BATCH_ENTRIES entries"); + assert!(agg.is_full()); + } + + #[test] + fn test_get_all_batches_splits_large_queue() { + use crate::constants::MAX_BATCH_ENTRIES; + let mut agg = Aggregator::new(); + let entry = make_entry("x"); + + for _ in 0..(MAX_BATCH_ENTRIES + 5) { + let _ = agg.insert(&entry); + } + + let batches = agg.get_all_batches(); + assert_eq!(batches.len(), 2, "should produce 2 batches"); + + let first: serde_json::Value = serde_json::from_slice(&batches[0]).expect("json"); + let second: serde_json::Value = serde_json::from_slice(&batches[1]).expect("json"); + assert_eq!(first.as_array().unwrap().len(), MAX_BATCH_ENTRIES); + assert_eq!(second.as_array().unwrap().len(), 5); + } + + #[test] + fn test_get_all_batches_empty_returns_empty_vec() { + let mut agg = Aggregator::new(); + assert!(agg.get_all_batches().is_empty()); + } + + #[test] + fn test_batch_never_exceeds_max_content_bytes() { + // Fill with entries whose sizes sum to just under MAX_CONTENT_BYTES so + // that the framing bytes (`[`, `]`, commas) would push a naive + // implementation over the limit. + let mut agg = Aggregator::new(); + // Each entry's serialized JSON is roughly 50 bytes; pack enough entries + // that their raw sum approaches MAX_CONTENT_BYTES. + let entry = make_entry("x"); + for _ in 0..1000 { + let _ = agg.insert(&entry); + } + + for batch in agg.get_all_batches() { + assert!( + batch.len() <= MAX_CONTENT_BYTES, + "batch size {} exceeds MAX_CONTENT_BYTES {}", + batch.len(), + MAX_CONTENT_BYTES + ); + } + } +} diff --git a/crates/datadog-logs-agent/src/aggregator/mod.rs b/crates/datadog-logs-agent/src/aggregator/mod.rs new file mode 100644 index 0000000..c191ab6 --- /dev/null +++ b/crates/datadog-logs-agent/src/aggregator/mod.rs @@ -0,0 +1,8 @@ +// Copyright 2025-Present Datadog, Inc. https://www.datadoghq.com/ +// SPDX-License-Identifier: Apache-2.0 + +pub mod core; +pub use core::Aggregator; + +pub mod service; +pub use service::{AggregatorHandle, AggregatorService}; diff --git a/crates/datadog-logs-agent/src/aggregator/service.rs b/crates/datadog-logs-agent/src/aggregator/service.rs new file mode 100644 index 0000000..c4ffc64 --- /dev/null +++ b/crates/datadog-logs-agent/src/aggregator/service.rs @@ -0,0 +1,185 @@ +// Copyright 2025-Present Datadog, Inc. https://www.datadoghq.com/ +// SPDX-License-Identifier: Apache-2.0 + +use tokio::sync::{mpsc, oneshot}; +use tracing::{debug, error, warn}; + +use crate::aggregator::Aggregator; +use crate::intake_entry::IntakeEntry; + +#[derive(Debug)] +enum AggregatorCommand { + InsertBatch(Vec), + GetBatches(oneshot::Sender>>), + Shutdown, +} + +/// Cloneable handle for sending commands to a running [`AggregatorService`]. +#[derive(Clone)] +pub struct AggregatorHandle { + tx: mpsc::UnboundedSender, +} + +impl AggregatorHandle { + /// Queue a batch of log entries for aggregation. + /// + /// Returns an error only if the service has already stopped. + pub fn insert_batch(&self, entries: Vec) -> Result<(), String> { + self.tx + .send(AggregatorCommand::InsertBatch(entries)) + .map_err(|e| format!("failed to send InsertBatch: {e}")) + } + + /// Retrieve and drain all accumulated log batches as JSON arrays. + /// + /// Returns an empty `Vec` if the aggregator holds no logs. + pub async fn get_batches(&self) -> Result>, String> { + let (tx, rx) = oneshot::channel(); + self.tx + .send(AggregatorCommand::GetBatches(tx)) + .map_err(|e| format!("failed to send GetBatches: {e}"))?; + rx.await + .map_err(|e| format!("failed to receive GetBatches response: {e}")) + } + + /// Signal the service to stop processing and exit its run loop. + pub fn shutdown(&self) -> Result<(), String> { + self.tx + .send(AggregatorCommand::Shutdown) + .map_err(|e| format!("failed to send Shutdown: {e}")) + } +} + +/// Background tokio task owning a [`Aggregator`] and processing commands. +/// +/// Create with [`AggregatorService::new`], spawn with `tokio::spawn(service.run())`, +/// and interact via the returned [`AggregatorHandle`]. +pub struct AggregatorService { + aggregator: Aggregator, + rx: mpsc::UnboundedReceiver, +} + +impl AggregatorService { + /// Create a new service and its associated handle. + pub fn new() -> (Self, AggregatorHandle) { + let (tx, rx) = mpsc::unbounded_channel(); + let service = Self { + aggregator: Aggregator::new(), + rx, + }; + let handle = AggregatorHandle { tx }; + (service, handle) + } + + /// Run the service event loop. + /// + /// Returns when a `Shutdown` command is received or the last handle is dropped. + pub async fn run(mut self) { + debug!("log aggregator service started"); + + while let Some(command) = self.rx.recv().await { + match command { + AggregatorCommand::InsertBatch(entries) => { + for entry in &entries { + if let Err(e) = self.aggregator.insert(entry) { + warn!("dropping log entry: {e}"); + } + } + } + + AggregatorCommand::GetBatches(response_tx) => { + let batches = self.aggregator.get_all_batches(); + if response_tx.send(batches).is_err() { + error!("failed to send GetBatches response — receiver dropped"); + } + } + + AggregatorCommand::Shutdown => { + debug!("log aggregator service shutting down"); + break; + } + } + } + + debug!("log aggregator service stopped"); + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::intake_entry::IntakeEntry; + + fn make_entry(msg: &str) -> IntakeEntry { + IntakeEntry::from_message(msg, 1_700_000_000_000) + } + + #[tokio::test] + async fn test_insert_and_get_batches_roundtrip() { + let (service, handle) = AggregatorService::new(); + let task = tokio::spawn(service.run()); + + handle + .insert_batch(vec![make_entry("a"), make_entry("b")]) + .expect("insert_batch failed"); + + let batches = handle.get_batches().await.expect("get_batches failed"); + assert_eq!(batches.len(), 1); + let arr: serde_json::Value = serde_json::from_slice(&batches[0]).expect("json"); + assert_eq!(arr.as_array().unwrap().len(), 2); + + handle.shutdown().expect("shutdown failed"); + task.await.expect("task panicked"); + } + + #[tokio::test] + async fn test_get_batches_empty_returns_empty_vec() { + let (service, handle) = AggregatorService::new(); + let task = tokio::spawn(service.run()); + + let batches = handle.get_batches().await.expect("get_batches"); + assert!(batches.is_empty()); + + handle.shutdown().expect("shutdown"); + task.await.expect("task"); + } + + #[tokio::test] + async fn test_oversized_entry_dropped_not_panicked() { + let (service, handle) = AggregatorService::new(); + let task = tokio::spawn(service.run()); + + let big = IntakeEntry::from_message("x".repeat(crate::constants::MAX_LOG_BYTES + 1), 0); + handle.insert_batch(vec![big]).expect("send ok"); + + let batches = handle.get_batches().await.expect("get_batches"); + assert!( + batches.is_empty(), + "oversized entry should have been dropped" + ); + + handle.shutdown().expect("shutdown"); + task.await.expect("task"); + } + + #[tokio::test] + async fn test_handle_is_clone_and_both_can_insert() { + let (service, handle) = AggregatorService::new(); + let task = tokio::spawn(service.run()); + + let handle2 = handle.clone(); + handle + .insert_batch(vec![make_entry("from h1")]) + .expect("h1"); + handle2 + .insert_batch(vec![make_entry("from h2")]) + .expect("h2"); + + let batches = handle.get_batches().await.expect("get_batches"); + let arr: serde_json::Value = serde_json::from_slice(&batches[0]).expect("json"); + assert_eq!(arr.as_array().unwrap().len(), 2); + + handle.shutdown().expect("shutdown"); + task.await.expect("task"); + } +} diff --git a/crates/datadog-logs-agent/src/config.rs b/crates/datadog-logs-agent/src/config.rs new file mode 100644 index 0000000..630e686 --- /dev/null +++ b/crates/datadog-logs-agent/src/config.rs @@ -0,0 +1,109 @@ +// Copyright 2025-Present Datadog, Inc. https://www.datadoghq.com/ +// SPDX-License-Identifier: Apache-2.0 + +use std::time::Duration; + +use crate::constants::{DEFAULT_COMPRESSION_LEVEL, DEFAULT_FLUSH_TIMEOUT_SECS, DEFAULT_SITE}; +use crate::logs_additional_endpoint::{LogsAdditionalEndpoint, parse_additional_endpoints}; + +/// Controls where and how logs are shipped. +#[derive(Debug, Clone)] +pub enum Destination { + /// Ship to Datadog Logs API. + /// Endpoint: `https://http-intake.logs.{site}/api/v2/logs` + /// Headers: `DD-API-KEY`, `DD-PROTOCOL: agent-json`, optionally `Content-Encoding: zstd` + Datadog, + + /// Ship to an Observability Pipelines Worker. + /// Endpoint: the provided URL. + /// Headers: `DD-API-KEY` only. Compression is always disabled for OPW. + ObservabilityPipelinesWorker { url: String }, +} + +/// Configuration for [`LogFlusher`](crate::flusher::LogFlusher). +#[derive(Debug, Clone)] +pub struct LogFlusherConfig { + /// Datadog API key. + pub api_key: String, + + /// Datadog site (e.g. "datadoghq.com", "datadoghq.eu"). + pub site: String, + + /// Flusher mode — Datadog vs Observability Pipelines Worker. + pub mode: Destination, + + /// Additional Datadog intake endpoints to ship each batch to in parallel. + /// Each endpoint uses its own API key and full intake URL. + pub additional_endpoints: Vec, + + /// Enable zstd compression (ignored in OPW mode, which is always uncompressed). + pub use_compression: bool, + + /// zstd compression level (ignored when `use_compression` is false). + pub compression_level: i32, + + /// Per-request timeout. + pub flush_timeout: Duration, +} + +impl LogFlusherConfig { + /// Build a config from environment variables, falling back to sensible defaults. + /// + /// | Variable | Default | + /// |---|---| + /// | `DD_API_KEY` | `""` | + /// | `DD_SITE` | `datadoghq.com` | + /// | `DD_LOGS_CONFIG_USE_COMPRESSION` | `true` | + /// | `DD_LOGS_CONFIG_COMPRESSION_LEVEL` | `3` | + /// | `DD_FLUSH_TIMEOUT` | `5` (seconds) | + /// | `DD_OBSERVABILITY_PIPELINES_WORKER_LOGS_ENABLED` | `false` | + /// | `DD_OBSERVABILITY_PIPELINES_WORKER_LOGS_URL` | (none) | + #[must_use] + pub fn from_env() -> Self { + let api_key = std::env::var("DD_API_KEY").unwrap_or_default(); + let site = std::env::var("DD_SITE").unwrap_or_else(|_| DEFAULT_SITE.to_string()); + + let use_compression = std::env::var("DD_LOGS_CONFIG_USE_COMPRESSION") + .map(|v| v.to_lowercase() != "false") + .unwrap_or(true); + + let compression_level = std::env::var("DD_LOGS_CONFIG_COMPRESSION_LEVEL") + .ok() + .and_then(|v| v.parse::().ok()) + .unwrap_or(DEFAULT_COMPRESSION_LEVEL); + + let flush_timeout_secs = std::env::var("DD_FLUSH_TIMEOUT") + .ok() + .and_then(|v| v.parse::().ok()) + .unwrap_or(DEFAULT_FLUSH_TIMEOUT_SECS); + + let opw_enabled = std::env::var("DD_OBSERVABILITY_PIPELINES_WORKER_LOGS_ENABLED") + .map(|v| v.to_lowercase() == "true") + .unwrap_or(false); + + let mode = if opw_enabled { + let url = + std::env::var("DD_OBSERVABILITY_PIPELINES_WORKER_LOGS_URL").unwrap_or_default(); + if url.is_empty() { + tracing::warn!( + "OPW mode enabled but DD_OBSERVABILITY_PIPELINES_WORKER_LOGS_URL is not set — log flush will fail" + ); + } + Destination::ObservabilityPipelinesWorker { url } + } else { + Destination::Datadog + }; + + Self { + api_key, + site, + mode, + additional_endpoints: std::env::var("DD_LOGS_CONFIG_ADDITIONAL_ENDPOINTS") + .map(|v| parse_additional_endpoints(&v)) + .unwrap_or_default(), + use_compression, + compression_level, + flush_timeout: Duration::from_secs(flush_timeout_secs), + } + } +} diff --git a/crates/datadog-logs-agent/src/constants.rs b/crates/datadog-logs-agent/src/constants.rs new file mode 100644 index 0000000..018dd51 --- /dev/null +++ b/crates/datadog-logs-agent/src/constants.rs @@ -0,0 +1,20 @@ +// Copyright 2025-Present Datadog, Inc. https://www.datadoghq.com/ +// SPDX-License-Identifier: Apache-2.0 + +/// Maximum number of log entries per batch. +pub const MAX_BATCH_ENTRIES: usize = 1_000; + +/// Maximum total uncompressed payload size per batch (5 MB). +pub const MAX_CONTENT_BYTES: usize = 5 * 1_024 * 1_024; + +/// Maximum allowed size for a single serialized log entry (1 MB). +pub const MAX_LOG_BYTES: usize = 1_024 * 1_024; + +/// Default Datadog site for log intake. +pub const DEFAULT_SITE: &str = "datadoghq.com"; + +/// Default flush timeout in seconds. +pub const DEFAULT_FLUSH_TIMEOUT_SECS: u64 = 5; + +/// Negative values enable ultra-fast modes. Level 3 is the zstd library default. +pub const DEFAULT_COMPRESSION_LEVEL: i32 = 3; diff --git a/crates/datadog-logs-agent/src/errors.rs b/crates/datadog-logs-agent/src/errors.rs new file mode 100644 index 0000000..50601af --- /dev/null +++ b/crates/datadog-logs-agent/src/errors.rs @@ -0,0 +1,35 @@ +// Copyright 2025-Present Datadog, Inc. https://www.datadoghq.com/ +// SPDX-License-Identifier: Apache-2.0 + +/// Errors that can occur when inserting log entries into the aggregator. +#[derive(Debug, thiserror::Error)] +pub enum AggregatorError { + #[error("log entry too large: {size} bytes exceeds max {max} bytes")] + EntryTooLarge { size: usize, max: usize }, + + #[error("failed to serialize log entry: {0}")] + Serialization(#[from] serde_json::Error), +} + +/// Errors that can occur when flushing logs to Datadog. +#[derive(Debug, thiserror::Error)] +pub enum FlushError { + #[error("HTTP request failed: {0}")] + Request(String), + + #[error("server returned permanent error: status {status}")] + PermanentError { status: u16 }, + + #[error("max retries exceeded after {attempts} attempts")] + MaxRetriesExceeded { attempts: u32 }, + + #[error("compression failed: {0}")] + Compression(String), +} + +/// Errors that can occur during crate object creation. +#[derive(Debug, thiserror::Error)] +pub enum CreationError { + #[error("failed to build HTTP client: {0}")] + HttpClient(String), +} diff --git a/crates/datadog-logs-agent/src/flusher.rs b/crates/datadog-logs-agent/src/flusher.rs new file mode 100644 index 0000000..933ac5e --- /dev/null +++ b/crates/datadog-logs-agent/src/flusher.rs @@ -0,0 +1,891 @@ +// Copyright 2025-Present Datadog, Inc. https://www.datadoghq.com/ +// SPDX-License-Identifier: Apache-2.0 + +use std::io::Write as _; + +use futures::future::join_all; +use reqwest::Client; +use tracing::{debug, error, warn}; +use zstd::stream::write::Encoder; + +use crate::aggregator::AggregatorHandle; +use crate::config::{Destination, LogFlusherConfig}; +use crate::errors::FlushError; + +/// Maximum number of send attempts before giving up on a batch. +const MAX_FLUSH_ATTEMPTS: u32 = 3; + +/// Drains log batches from an [`AggregatorHandle`] and ships them to Datadog. +#[derive(Clone)] +pub struct LogFlusher { + config: LogFlusherConfig, + client: Client, + aggregator_handle: AggregatorHandle, +} + +impl LogFlusher { + /// Create a new flusher. + /// + /// The `client` **must** be built via + /// [`datadog_fips::reqwest_adapter::create_reqwest_client_builder`] to ensure + /// FIPS-compliant TLS. Never use `reqwest::Client::builder()` directly. + pub fn new( + config: LogFlusherConfig, + client: Client, + aggregator_handle: AggregatorHandle, + ) -> Self { + Self { + config, + client, + aggregator_handle, + } + } + + /// Drain the aggregator, ship all pending batches to Datadog, and redrive any + /// builders that failed transiently in the previous invocation. + /// + /// # Arguments + /// + /// * `retry_requests` — builders returned by a previous `flush` call that + /// exhausted their per-invocation retry budget. They are re-sent before + /// draining new batches from the aggregator. + /// + /// # Returns + /// + /// A vec of `RequestBuilder`s that still failed after all in-call retries. + /// The caller should pass these back on the next invocation to re-attempt + /// delivery. An empty vec means every batch was delivered successfully + /// (or encountered a permanent error and was dropped — those are logged). + /// + /// Failures on additional endpoints are logged as warnings but their + /// builders are not included in the returned vec (best-effort delivery). + pub async fn flush( + &self, + retry_requests: Vec, + ) -> Vec { + let mut failed: Vec = Vec::new(); + + // Redrive builders that failed transiently in the previous invocation. + if !retry_requests.is_empty() { + debug!( + "redriving {} log builder(s) from previous flush", + retry_requests.len() + ); + } + let retry_futures = retry_requests + .into_iter() + .map(|builder| async move { self.send_with_retry(builder).await.err() }); + for b in join_all(retry_futures).await.into_iter().flatten() { + failed.push(b); + } + + // Drain new batches from the aggregator. + let batches = match self.aggregator_handle.get_batches().await { + Ok(b) => b, + Err(e) => { + error!("failed to retrieve log batches from aggregator: {e}"); + return failed; + } + }; + + if batches.is_empty() { + debug!("no log batches to flush"); + return failed; + } + + debug!("flushing {} log batch(es)", batches.len()); + + let (primary_url, primary_use_compression) = self.resolve_endpoint(); + + let batch_futures = batches.iter().map(|batch| { + let primary_url = primary_url.clone(); + async move { + // Primary endpoint — failures are tracked for cross-invocation retry. + let is_primary_datadog = matches!(self.config.mode, Destination::Datadog); + let primary_result = self + .ship_batch(batch, &primary_url, primary_use_compression, &self.config.api_key, is_primary_datadog) + .await; + + // Additional endpoints — best-effort; failures are only logged. + // Additional endpoints are always Datadog intakes regardless of the + // primary destination, so DD-PROTOCOL: agent-json is always required. + // They use config.use_compression independently of the primary: OPW + // primaries disable compression for themselves but Datadog extras + // should still honour DD_LOGS_CONFIG_USE_COMPRESSION. + let extra_futures = self.config.additional_endpoints.iter().map(|endpoint| { + let url = endpoint.url.clone(); + let api_key = endpoint.api_key.clone(); + async move { + if self + .ship_batch(batch, &url, self.config.use_compression, &api_key, true) + .await + .is_err() + { + warn!( + "failed to ship log batch to additional endpoint {url} after all retries" + ); + } + } + }); + join_all(extra_futures).await; + + primary_result + } + }); + + for result in join_all(batch_futures).await { + if let Err(b) = result { + failed.push(b); + } + } + + failed + } + + fn resolve_endpoint(&self) -> (String, bool) { + match &self.config.mode { + Destination::Datadog => { + let url = format!("https://http-intake.logs.{}/api/v2/logs", self.config.site); + (url, self.config.use_compression) + } + Destination::ObservabilityPipelinesWorker { url } => { + // OPW does not support compression + (url.clone(), false) + } + } + } + + async fn ship_batch( + &self, + batch: &[u8], + url: &str, + compress: bool, + api_key: &str, + is_datadog_intake: bool, + ) -> Result<(), reqwest::RequestBuilder> { + let (body, content_encoding) = if compress { + match compress_zstd(batch, self.config.compression_level) { + Ok(compressed) => (compressed, Some("zstd")), + Err(e) => { + warn!("failed to compress log batch, sending uncompressed: {e}"); + (batch.to_vec(), None) + } + } + } else { + (batch.to_vec(), None) + }; + + let mut req = self + .client + .post(url) + .timeout(self.config.flush_timeout) + .header("DD-API-KEY", api_key) + .header("Content-Type", "application/json"); + + if is_datadog_intake { + req = req.header("DD-PROTOCOL", "agent-json"); + } + + if let Some(enc) = content_encoding { + req = req.header("Content-Encoding", enc); + } + + let req = req.body(body); + self.send_with_retry(req).await + } + + /// Send `builder`, retrying transient failures up to `MAX_FLUSH_ATTEMPTS`. + /// + /// # Returns + /// + /// * `Ok(())` — success **or** a permanent error (no point retrying; already + /// logged at `warn!`). + /// * `Err(builder)` — all attempts exhausted on a transient error. The + /// original builder is returned so the caller can retry it next invocation. + async fn send_with_retry( + &self, + builder: reqwest::RequestBuilder, + ) -> Result<(), reqwest::RequestBuilder> { + let mut attempts: u32 = 0; + + loop { + attempts += 1; + + let cloned = match builder.try_clone() { + Some(b) => b, + None => { + // Streaming body — can't clone, can't retry. + warn!("log batch request is not cloneable; dropping batch"); + return Ok(()); + } + }; + + match cloned.send().await { + Ok(resp) => { + let status = resp.status(); + // Drain the body so the underlying TCP connection is + // returned to the pool rather than held in CLOSE_WAIT. + let _ = resp.bytes().await; + + if status.is_success() { + debug!("log batch accepted: {status}"); + return Ok(()); + } + + // Retryable 4xx: treat like transient server errors and + // fall through to the retry loop below. + // 408 = Request Timeout (transient network condition) + // 425 = Too Early (TLS 0-RTT replay rejection) + // 429 = Too Many Requests (intake rate-limiting) + // + // TODO: for 429, parse the `Retry-After` response header + // and sleep for the indicated duration before retrying + // instead of retrying immediately, to avoid hammering the + // intake endpoint while it is still rate-limiting us. + let retryable_4xx = matches!(status.as_u16(), 408 | 425 | 429); + + // Permanent client errors — stop immediately, do not retry. + if status.as_u16() >= 400 && status.as_u16() < 500 && !retryable_4xx { + warn!("permanent error from logs intake: {status}; dropping batch"); + return Ok(()); + } + + // Transient server errors — fall through to retry. + warn!( + "transient error from logs intake: {status} (attempt {attempts}/{MAX_FLUSH_ATTEMPTS})" + ); + } + Err(e) => { + warn!( + "network error sending log batch (attempt {attempts}/{MAX_FLUSH_ATTEMPTS}): {e}" + ); + } + } + + if attempts >= MAX_FLUSH_ATTEMPTS { + warn!("log batch failed after {attempts} attempts; will retry next flush"); + return Err(builder); + } + } + } +} + +fn compress_zstd(data: &[u8], level: i32) -> Result, FlushError> { + let mut encoder = + Encoder::new(Vec::new(), level).map_err(|e| FlushError::Compression(e.to_string()))?; + encoder + .write_all(data) + .map_err(|e| FlushError::Compression(e.to_string()))?; + encoder + .finish() + .map_err(|e| FlushError::Compression(e.to_string())) +} + +#[cfg(test)] +// Tests use plain reqwest client to connect to local mock server +#[allow(clippy::disallowed_methods)] +mod tests { + use super::*; + use crate::aggregator::AggregatorService; + use crate::config::{Destination, LogFlusherConfig}; + use crate::intake_entry::IntakeEntry; + use crate::logs_additional_endpoint::LogsAdditionalEndpoint; + use mockito::Matcher; + use std::time::Duration; + + fn make_entry(msg: &str) -> IntakeEntry { + IntakeEntry::from_message(msg, 1_700_000_000_000) + } + + fn config_for_mock(mock_url: &str) -> LogFlusherConfig { + // Use OPW mode pointing at the mock server to avoid HTTPS + LogFlusherConfig { + api_key: "test-api-key".to_string(), + site: "datadoghq.com".to_string(), + mode: Destination::ObservabilityPipelinesWorker { + url: format!("{mock_url}/api/v2/logs"), + }, + additional_endpoints: Vec::new(), + use_compression: false, + compression_level: 3, + flush_timeout: Duration::from_secs(5), + } + } + + #[tokio::test] + async fn test_flush_empty_aggregator_does_not_call_api() { + let (service, handle) = AggregatorService::new(); + let _task = tokio::spawn(service.run()); + + // Server with no routes — any request would cause test failure + let mock_server = mockito::Server::new_async().await; + let config = config_for_mock(&mock_server.url()); + let client = reqwest::Client::builder().build().expect("client"); + let flusher = LogFlusher::new(config, client, handle); + + assert!( + flusher.flush(vec![]).await.is_empty(), + "empty flush should succeed" + ); + // No mock assertions needed — absence of request is the assertion + } + + #[tokio::test] + async fn test_flush_sends_post_with_api_key_header() { + // Verify that Datadog mode sends both DD-API-KEY and DD-PROTOCOL: + // agent-json headers. We call ship_batch directly to bypass + // resolve_endpoint (which builds an HTTPS URL incompatible with the + // HTTP mock server). + let (service, handle) = AggregatorService::new(); + let _task = tokio::spawn(service.run()); + + let mut mock_server = mockito::Server::new_async().await; + let mock = mock_server + .mock("POST", "/api/v2/logs") + .match_header("DD-API-KEY", "test-api-key") + .match_header("DD-PROTOCOL", "agent-json") + .with_status(202) + .create_async() + .await; + + let config = LogFlusherConfig { + api_key: "test-api-key".to_string(), + site: "datadoghq.com".to_string(), + mode: Destination::Datadog, + additional_endpoints: Vec::new(), + use_compression: false, + compression_level: 3, + flush_timeout: Duration::from_secs(5), + }; + + let client = reqwest::Client::builder().build().expect("client"); + let flusher = LogFlusher::new(config, client, handle); + + // Call ship_batch directly to use the mock server's HTTP URL instead + // of the HTTPS URL that resolve_endpoint would produce. + let url = format!("{}/api/v2/logs", mock_server.url()); + let batch = b"[{\"message\":\"test\"}]"; + flusher + .ship_batch(batch, &url, false, "test-api-key", true) + .await + .expect("ship_batch should succeed"); + + mock.assert_async().await; + } + + #[tokio::test] + async fn test_flush_opw_mode_omits_dd_protocol_header() { + let (service, handle) = AggregatorService::new(); + let _task = tokio::spawn(service.run()); + + let mut mock_server = mockito::Server::new_async().await; + let opw_url = format!("{}/logs", mock_server.url()); + + // Verify DD-PROTOCOL is NOT present in OPW requests + let mock = mock_server + .mock("POST", "/logs") + .match_header("DD-API-KEY", "test-api-key") + .match_header("DD-PROTOCOL", Matcher::Missing) + .with_status(200) + .expect(1) + .create_async() + .await; + + let config = LogFlusherConfig { + api_key: "test-api-key".to_string(), + site: "unused".to_string(), + mode: Destination::ObservabilityPipelinesWorker { url: opw_url }, + additional_endpoints: Vec::new(), + use_compression: false, + compression_level: 3, + flush_timeout: Duration::from_secs(5), + }; + + let client = reqwest::Client::builder().build().expect("client"); + let flusher = LogFlusher::new(config, client, handle.clone()); + + handle + .insert_batch(vec![make_entry("opw log")]) + .expect("insert"); + let result = flusher.flush(vec![]).await; + assert!(result.is_empty(), "OPW flush should return empty on 200"); + mock.assert_async().await; + } + + /// When the primary destination is OPW, additional endpoints are still Datadog + /// intakes and must receive the `DD-PROTOCOL: agent-json` header. + #[tokio::test] + async fn test_opw_primary_additional_endpoint_receives_dd_protocol_header() { + let (service, handle) = AggregatorService::new(); + let _task = tokio::spawn(service.run()); + + let mut primary = mockito::Server::new_async().await; + let mut extra = mockito::Server::new_async().await; + + // OPW primary must NOT have DD-PROTOCOL header + let _primary_mock = primary + .mock("POST", "/logs") + .match_header("DD-PROTOCOL", Matcher::Missing) + .with_status(200) + .expect(1) + .create_async() + .await; + + // Additional endpoint (Datadog intake) MUST have DD-PROTOCOL header + let extra_mock = extra + .mock("POST", "/extra") + .match_header("DD-PROTOCOL", "agent-json") + .with_status(202) + .expect(1) + .create_async() + .await; + + let config = LogFlusherConfig { + api_key: "key".to_string(), + site: "datadoghq.com".to_string(), + mode: Destination::ObservabilityPipelinesWorker { + url: format!("{}/logs", primary.url()), + }, + additional_endpoints: vec![LogsAdditionalEndpoint { + api_key: "extra-key".to_string(), + url: format!("{}/extra", extra.url()), + is_reliable: true, + }], + use_compression: false, + compression_level: 3, + flush_timeout: Duration::from_secs(5), + }; + + let client = reqwest::Client::builder().build().expect("client"); + let flusher = LogFlusher::new(config, client, handle.clone()); + handle + .insert_batch(vec![make_entry("test")]) + .expect("insert"); + + assert!(flusher.flush(vec![]).await.is_empty()); + extra_mock.assert_async().await; + } + + #[tokio::test] + async fn test_flush_does_not_retry_on_403() { + let (service, handle) = AggregatorService::new(); + let _task = tokio::spawn(service.run()); + + let mut mock_server = mockito::Server::new_async().await; + // expect(1) means exactly one call — if retried, the test will fail + let mock = mock_server + .mock("POST", "/api/v2/logs") + .with_status(403) + .expect(1) + .create_async() + .await; + + let config = config_for_mock(&mock_server.url()); + let client = reqwest::Client::builder().build().expect("client"); + let flusher = LogFlusher::new(config, client, handle.clone()); + + handle + .insert_batch(vec![make_entry("log")]) + .expect("insert"); + let result = flusher.flush(vec![]).await; + // 403 is a permanent error — the batch is dropped, no builder to retry. + assert!( + result.is_empty(), + "403 is a permanent error; no builder to retry" + ); + mock.assert_async().await; + } + + /// 429 (Too Many Requests) is a retryable 4xx — the retry loop must + /// continue rather than short-circuiting with a permanent failure. + #[tokio::test] + async fn test_flush_retries_on_429_then_succeeds() { + let (service, handle) = AggregatorService::new(); + let _task = tokio::spawn(service.run()); + + let mut mock_server = mockito::Server::new_async().await; + // First call → 429, second call → 200 + let _throttled = mock_server + .mock("POST", "/api/v2/logs") + .with_status(429) + .expect(1) + .create_async() + .await; + let _ok = mock_server + .mock("POST", "/api/v2/logs") + .with_status(200) + .expect(1) + .create_async() + .await; + + let config = config_for_mock(&mock_server.url()); + let client = reqwest::Client::builder().build().expect("client"); + let flusher = LogFlusher::new(config, client, handle.clone()); + + handle + .insert_batch(vec![make_entry("throttled log")]) + .expect("insert"); + let result = flusher.flush(vec![]).await; + assert!(result.is_empty(), "should succeed after 429 retry"); + } + + #[tokio::test] + async fn test_flush_retries_on_5xx_then_succeeds() { + let (service, handle) = AggregatorService::new(); + let _task = tokio::spawn(service.run()); + + let mut mock_server = mockito::Server::new_async().await; + // First call → 500, second call → 202 + let _fail_mock = mock_server + .mock("POST", "/api/v2/logs") + .with_status(500) + .expect(1) + .create_async() + .await; + let _ok_mock = mock_server + .mock("POST", "/api/v2/logs") + .with_status(202) + .expect(1) + .create_async() + .await; + + let config = config_for_mock(&mock_server.url()); + let client = reqwest::Client::builder().build().expect("client"); + let flusher = LogFlusher::new(config, client, handle.clone()); + + handle + .insert_batch(vec![make_entry("log")]) + .expect("insert"); + let result = flusher.flush(vec![]).await; + assert!(result.is_empty(), "should succeed on second attempt"); + } + + /// All additional endpoints receive the same batch when flush() is called. + #[tokio::test] + async fn test_additional_endpoints_all_receive_batch() { + let (service, handle) = AggregatorService::new(); + let _task = tokio::spawn(service.run()); + + let mut primary = mockito::Server::new_async().await; + let mut extra1 = mockito::Server::new_async().await; + let mut extra2 = mockito::Server::new_async().await; + + let primary_mock = primary + .mock("POST", "/api/v2/logs") + .with_status(202) + .expect(1) + .create_async() + .await; + let extra1_mock = extra1 + .mock("POST", "/extra") + .with_status(200) + .expect(1) + .create_async() + .await; + let extra2_mock = extra2 + .mock("POST", "/extra") + .with_status(200) + .expect(1) + .create_async() + .await; + + let config = LogFlusherConfig { + api_key: "key".to_string(), + site: "datadoghq.com".to_string(), + mode: Destination::ObservabilityPipelinesWorker { + url: format!("{}/api/v2/logs", primary.url()), + }, + additional_endpoints: vec![ + LogsAdditionalEndpoint { + api_key: "extra-key-1".to_string(), + url: format!("{}/extra", extra1.url()), + is_reliable: true, + }, + LogsAdditionalEndpoint { + api_key: "extra-key-2".to_string(), + url: format!("{}/extra", extra2.url()), + is_reliable: true, + }, + ], + use_compression: false, + compression_level: 3, + flush_timeout: Duration::from_secs(5), + }; + + let client = reqwest::Client::builder().build().expect("client"); + let flusher = LogFlusher::new(config, client, handle.clone()); + handle.insert_batch(vec![make_entry("hi")]).expect("insert"); + + assert!(flusher.flush(vec![]).await.is_empty()); + primary_mock.assert_async().await; + extra1_mock.assert_async().await; + extra2_mock.assert_async().await; + } + + /// Additional endpoints are dispatched concurrently: if they were sequential, + /// two endpoints each waiting at a Barrier(2) would deadlock — only concurrent + /// dispatch lets both handlers reach the barrier simultaneously. + #[tokio::test] + async fn test_additional_endpoints_dispatched_concurrently() { + use std::sync::Arc; + use tokio::io::{AsyncReadExt as _, AsyncWriteExt as _}; + use tokio::net::TcpListener; + use tokio::sync::Barrier; + + let (service, handle) = AggregatorService::new(); + let _task = tokio::spawn(service.run()); + + let barrier = Arc::new(Barrier::new(2)); + + // Spawn a minimal HTTP server that waits at the barrier before + // responding, so both must be in-flight at the same time to complete. + async fn serve_once(barrier: Arc) -> String { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = listener.local_addr().unwrap(); + tokio::spawn(async move { + let (mut stream, _) = listener.accept().await.unwrap(); + let mut buf = vec![0u8; 4096]; + let _ = stream.read(&mut buf).await; + barrier.wait().await; + let _ = stream + .write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 0\r\nConnection: close\r\n\r\n") + .await; + }); + format!("http://127.0.0.1:{}/logs", addr.port()) + } + + let url1 = serve_once(barrier.clone()).await; + let url2 = serve_once(barrier.clone()).await; + + let mut primary = mockito::Server::new_async().await; + let _primary_mock = primary + .mock("POST", "/api/v2/logs") + .with_status(202) + .create_async() + .await; + + let config = LogFlusherConfig { + api_key: "key".to_string(), + site: "datadoghq.com".to_string(), + mode: Destination::ObservabilityPipelinesWorker { + url: format!("{}/api/v2/logs", primary.url()), + }, + additional_endpoints: vec![ + LogsAdditionalEndpoint { + api_key: "extra-key-1".to_string(), + url: url1, + is_reliable: true, + }, + LogsAdditionalEndpoint { + api_key: "extra-key-2".to_string(), + url: url2, + is_reliable: true, + }, + ], + use_compression: false, + compression_level: 3, + flush_timeout: Duration::from_secs(5), + }; + + let client = reqwest::Client::builder().build().expect("client"); + let flusher = LogFlusher::new(config, client, handle.clone()); + handle + .insert_batch(vec![make_entry("concurrent")]) + .expect("insert"); + + assert!(flusher.flush(vec![]).await.is_empty()); + } + + /// A builder returned by `flush` can be redriven on the next call. + /// + /// The mock fails on the first 3 attempts (exhausting the per-invocation + /// retry budget), then succeeds on the 4th attempt (the next invocation). + /// This proves the cross-invocation retry path end-to-end. + #[tokio::test] + async fn test_cross_invocation_retry_delivers_on_redrive() { + let (service, handle) = AggregatorService::new(); + let _task = tokio::spawn(service.run()); + + let mut mock_server = mockito::Server::new_async().await; + // First 3 calls: transient 503 → exhausts per-invocation retry budget + let _fail_mock = mock_server + .mock("POST", "/api/v2/logs") + .with_status(503) + .expect(3) + .create_async() + .await; + // 4th call: redriven on the next flush → succeeds + let _ok_mock = mock_server + .mock("POST", "/api/v2/logs") + .with_status(200) + .expect(1) + .create_async() + .await; + + let config = config_for_mock(&mock_server.url()); + let client = reqwest::Client::builder().build().expect("client"); + let flusher = LogFlusher::new(config, client, handle.clone()); + handle + .insert_batch(vec![make_entry("retry-me")]) + .expect("insert"); + + // First flush: all 3 attempts fail → returns the builder for retry. + let failed = flusher.flush(vec![]).await; + assert_eq!(failed.len(), 1, "one builder should be returned for retry"); + + // Second flush: aggregator is empty; redrives the failed builder → succeeds. + let result = flusher.flush(failed).await; + assert!( + result.is_empty(), + "redriven builder should succeed on the next invocation" + ); + } + + /// When the primary is OPW (compression disabled for OPW transport) and + /// `use_compression` is true, additional Datadog endpoints must still + /// receive `Content-Encoding: zstd` — they are Datadog intakes and honour + /// `DD_LOGS_CONFIG_USE_COMPRESSION` independently of the primary. + #[tokio::test] + async fn test_opw_primary_additional_endpoint_compresses_when_enabled() { + let (service, handle) = AggregatorService::new(); + let _task = tokio::spawn(service.run()); + + let mut primary = mockito::Server::new_async().await; + let mut extra = mockito::Server::new_async().await; + + // OPW primary must NOT have Content-Encoding + let _primary_mock = primary + .mock("POST", "/logs") + .match_header("Content-Encoding", Matcher::Missing) + .with_status(200) + .expect(1) + .create_async() + .await; + + // Additional Datadog endpoint MUST receive Content-Encoding: zstd + let extra_mock = extra + .mock("POST", "/extra") + .match_header("Content-Encoding", "zstd") + .with_status(202) + .expect(1) + .create_async() + .await; + + let config = LogFlusherConfig { + api_key: "key".to_string(), + site: "datadoghq.com".to_string(), + mode: Destination::ObservabilityPipelinesWorker { + url: format!("{}/logs", primary.url()), + }, + additional_endpoints: vec![LogsAdditionalEndpoint { + api_key: "extra-key".to_string(), + url: format!("{}/extra", extra.url()), + is_reliable: true, + }], + use_compression: true, + compression_level: 3, + flush_timeout: Duration::from_secs(5), + }; + + let client = reqwest::Client::builder().build().expect("client"); + let flusher = LogFlusher::new(config, client, handle.clone()); + handle + .insert_batch(vec![make_entry("compressed extra")]) + .expect("insert"); + + assert!(flusher.flush(vec![]).await.is_empty()); + extra_mock.assert_async().await; + } + + /// Even when `use_compression: true`, the OPW primary endpoint must never + /// receive a `Content-Encoding` header — OPW does not support zstd. + #[tokio::test] + async fn test_opw_primary_never_compressed_even_when_flag_set() { + let (service, handle) = AggregatorService::new(); + let _task = tokio::spawn(service.run()); + + let mut mock_server = mockito::Server::new_async().await; + let mock = mock_server + .mock("POST", "/logs") + .match_header("Content-Encoding", Matcher::Missing) + .with_status(200) + .expect(1) + .create_async() + .await; + + let config = LogFlusherConfig { + api_key: "key".to_string(), + site: "datadoghq.com".to_string(), + mode: Destination::ObservabilityPipelinesWorker { + url: format!("{}/logs", mock_server.url()), + }, + additional_endpoints: vec![], + use_compression: true, // flag set but must not reach OPW + compression_level: 3, + flush_timeout: Duration::from_secs(5), + }; + + let client = reqwest::Client::builder().build().expect("client"); + let flusher = LogFlusher::new(config, client, handle.clone()); + handle + .insert_batch(vec![make_entry("opw no compress")]) + .expect("insert"); + + assert!(flusher.flush(vec![]).await.is_empty()); + mock.assert_async().await; + } + + /// Additional-endpoint failures are best-effort: their builders are NOT + /// included in the returned retry vec, even when they exhaust all retries. + /// Only primary-endpoint failures are tracked for cross-invocation retry. + #[tokio::test] + async fn test_additional_endpoint_failures_not_tracked_for_retry() { + let (service, handle) = AggregatorService::new(); + let _task = tokio::spawn(service.run()); + + let mut primary = mockito::Server::new_async().await; + let mut extra = mockito::Server::new_async().await; + + let _primary_mock = primary + .mock("POST", "/api/v2/logs") + .with_status(200) + .expect(1) + .create_async() + .await; + // Additional endpoint always returns 503 — exhausts per-invocation retries. + let _extra_mock = extra + .mock("POST", "/extra") + .with_status(503) + .expect(3) // MAX_FLUSH_ATTEMPTS + .create_async() + .await; + + let config = LogFlusherConfig { + api_key: "key".to_string(), + site: "datadoghq.com".to_string(), + mode: Destination::ObservabilityPipelinesWorker { + url: format!("{}/api/v2/logs", primary.url()), + }, + additional_endpoints: vec![LogsAdditionalEndpoint { + api_key: "extra-key".to_string(), + url: format!("{}/extra", extra.url()), + is_reliable: true, + }], + use_compression: false, + compression_level: 3, + flush_timeout: Duration::from_secs(5), + }; + + let client = reqwest::Client::builder().build().expect("client"); + let flusher = LogFlusher::new(config, client, handle.clone()); + handle + .insert_batch(vec![make_entry("test")]) + .expect("insert"); + + let result = flusher.flush(vec![]).await; + assert!( + result.is_empty(), + "additional-endpoint failures are best-effort and must not be tracked for retry" + ); + } +} diff --git a/crates/datadog-logs-agent/src/intake_entry.rs b/crates/datadog-logs-agent/src/intake_entry.rs new file mode 100644 index 0000000..2ec0186 --- /dev/null +++ b/crates/datadog-logs-agent/src/intake_entry.rs @@ -0,0 +1,188 @@ +// Copyright 2025-Present Datadog, Inc. https://www.datadoghq.com/ +// SPDX-License-Identifier: Apache-2.0 + +use serde::{Deserialize, Serialize}; + +/// A single log entry in the Datadog Logs intake format. +/// +/// Standard Datadog fields are typed fields. Runtime-specific enrichment +/// (e.g. `{"lambda": {"arn": "...", "request_id": "..."}}` for Lambda, +/// `{"azure": {"resource_id": "..."}}` for Azure Functions) goes in `attributes`, +/// which is flattened into the JSON object at serialization time. +/// +/// # Example — Lambda extension consumer +/// ```ignore +/// let mut attrs = serde_json::Map::new(); +/// attrs.insert("lambda".to_string(), serde_json::json!({ +/// "arn": function_arn, +/// "request_id": request_id, +/// })); +/// let entry = IntakeEntry { +/// message: log_line, +/// timestamp: timestamp_ms, +/// hostname: Some(function_arn), +/// service: Some(service_name), +/// ddsource: Some("lambda".to_string()), +/// ddtags: Some(tags), +/// status: Some("info".to_string()), +/// attributes: attrs, +/// }; +/// ``` +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct IntakeEntry { + /// The log message body. + pub message: String, + + /// Unix timestamp in milliseconds. + pub timestamp: i64, + + /// The hostname (e.g. Lambda function ARN, Azure resource ID). + #[serde(skip_serializing_if = "Option::is_none")] + pub hostname: Option, + + /// The service name. + #[serde(skip_serializing_if = "Option::is_none")] + pub service: Option, + + /// The log source tag (e.g. "lambda", "azure-functions", "gcp-functions"). + #[serde(skip_serializing_if = "Option::is_none")] + pub ddsource: Option, + + /// Comma-separated Datadog tags (e.g. "env:prod,version:1.0"). + #[serde(skip_serializing_if = "Option::is_none")] + pub ddtags: Option, + + /// Log level / status (e.g. "info", "error", "warn", "debug"). + #[serde(skip_serializing_if = "Option::is_none")] + pub status: Option, + + /// Runtime-specific enrichment fields, flattened into the JSON object. + /// Use this for fields like `{"lambda": {"arn": "...", "request_id": "..."}}`. + /// An empty map is not serialized. Extra fields in JSON are collected here on deserialization. + #[serde(flatten, default, skip_serializing_if = "serde_json::Map::is_empty")] + pub attributes: serde_json::Map, +} + +impl IntakeEntry { + /// Create a minimal log entry from a message and timestamp. + /// All optional fields default to `None`; use struct literal syntax to set them. + pub fn from_message(message: impl Into, timestamp: i64) -> Self { + Self { + message: message.into(), + timestamp, + hostname: None, + service: None, + ddsource: None, + ddtags: None, + status: None, + attributes: serde_json::Map::new(), + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_intake_entry_minimal_serialization() { + let entry = IntakeEntry::from_message("hello world", 1_700_000_000_000); + let json = serde_json::to_string(&entry).expect("serialize"); + let v: serde_json::Value = serde_json::from_str(&json).expect("parse"); + + assert_eq!(v["message"], "hello world"); + assert_eq!(v["timestamp"], 1_700_000_000_000_i64); + // Optional fields must be absent when None + assert!(v.get("hostname").is_none()); + assert!(v.get("service").is_none()); + assert!(v.get("ddsource").is_none()); + assert!(v.get("ddtags").is_none()); + assert!(v.get("status").is_none()); + } + + #[test] + fn test_intake_entry_full_serialization() { + let entry = IntakeEntry { + message: "user logged in".to_string(), + timestamp: 1_700_000_001_000, + hostname: Some("my-host".to_string()), + service: Some("my-service".to_string()), + ddsource: Some("lambda".to_string()), + ddtags: Some("env:prod,version:1.0".to_string()), + status: Some("info".to_string()), + attributes: serde_json::Map::new(), + }; + let json = serde_json::to_string(&entry).expect("serialize"); + let v: serde_json::Value = serde_json::from_str(&json).expect("parse"); + + assert_eq!(v["message"], "user logged in"); + assert_eq!(v["hostname"], "my-host"); + assert_eq!(v["service"], "my-service"); + assert_eq!(v["ddsource"], "lambda"); + assert_eq!(v["ddtags"], "env:prod,version:1.0"); + assert_eq!(v["status"], "info"); + assert!( + v.get("attributes").is_none(), + "empty attributes must not appear in output" + ); + } + + #[test] + fn test_intake_entry_with_lambda_attributes_flattened() { + // Simulates what the lambda extension would build + let mut attrs = serde_json::Map::new(); + attrs.insert( + "lambda".to_string(), + serde_json::json!({ + "arn": "arn:aws:lambda:us-east-1:123456789012:function:my-fn", + "request_id": "abc-123" + }), + ); + let entry = IntakeEntry { + message: "function invoked".to_string(), + timestamp: 1_700_000_002_000, + hostname: Some("arn:aws:lambda:us-east-1:123456789012:function:my-fn".to_string()), + service: Some("my-fn".to_string()), + ddsource: Some("lambda".to_string()), + ddtags: Some("env:prod".to_string()), + status: Some("info".to_string()), + attributes: attrs, + }; + let json = serde_json::to_string(&entry).expect("serialize"); + let v: serde_json::Value = serde_json::from_str(&json).expect("parse"); + + // Lambda-specific fields appear at top level (flattened) + assert_eq!( + v["lambda"]["arn"], + "arn:aws:lambda:us-east-1:123456789012:function:my-fn" + ); + assert_eq!(v["lambda"]["request_id"], "abc-123"); + assert_eq!(v["message"], "function invoked"); + } + + #[test] + fn test_intake_entry_deserialization_roundtrip() { + let original = IntakeEntry { + message: "test".to_string(), + timestamp: 42, + hostname: Some("h".to_string()), + service: None, + ddsource: Some("gcp-functions".to_string()), + ddtags: None, + status: Some("error".to_string()), + attributes: serde_json::Map::new(), + }; + let json = serde_json::to_string(&original).expect("serialize"); + let restored: IntakeEntry = serde_json::from_str(&json).expect("deserialize"); + + assert_eq!(restored.message, original.message); + assert_eq!(restored.timestamp, original.timestamp); + assert_eq!(restored.hostname, original.hostname); + assert_eq!(restored.ddsource, original.ddsource); + assert_eq!(restored.status, original.status); + assert!( + restored.attributes.is_empty(), + "no extra attributes expected after roundtrip" + ); + } +} diff --git a/crates/datadog-logs-agent/src/lib.rs b/crates/datadog-logs-agent/src/lib.rs new file mode 100644 index 0000000..ef1c454 --- /dev/null +++ b/crates/datadog-logs-agent/src/lib.rs @@ -0,0 +1,26 @@ +// Copyright 2023-Present Datadog, Inc. https://www.datadoghq.com/ +// SPDX-License-Identifier: Apache-2.0 + +#![cfg_attr(not(test), deny(clippy::panic))] +#![cfg_attr(not(test), deny(clippy::unwrap_used))] +#![cfg_attr(not(test), deny(clippy::expect_used))] +#![cfg_attr(not(test), deny(clippy::todo))] +#![cfg_attr(not(test), deny(clippy::unimplemented))] + +pub mod aggregator; +pub mod config; +pub mod constants; +pub mod errors; +pub mod flusher; +pub mod intake_entry; +pub mod logs_additional_endpoint; + +pub mod server; + +// Re-export the most commonly used types at the crate root +pub use aggregator::{AggregatorHandle, AggregatorService}; +pub use config::{Destination, LogFlusherConfig}; +pub use flusher::LogFlusher; +pub use intake_entry::IntakeEntry; +pub use logs_additional_endpoint::LogsAdditionalEndpoint; +pub use server::{LogServer, LogServerConfig}; diff --git a/crates/datadog-logs-agent/src/logs_additional_endpoint.rs b/crates/datadog-logs-agent/src/logs_additional_endpoint.rs new file mode 100644 index 0000000..9413e0b --- /dev/null +++ b/crates/datadog-logs-agent/src/logs_additional_endpoint.rs @@ -0,0 +1,104 @@ +// Copyright 2025-Present Datadog, Inc. https://www.datadoghq.com/ +// SPDX-License-Identifier: Apache-2.0 + +use serde::Deserialize; +use tracing::warn; + +/// An additional Datadog intake endpoint to ship each log batch to alongside +/// the primary endpoint. +/// +/// The JSON wire format (from `DD_LOGS_CONFIG_ADDITIONAL_ENDPOINTS`) matches the +/// bottlecap / datadog-agent convention: +/// ```json +/// [{"api_key":"","Host":"agent-http-intake.logs.datadoghq.com","Port":443,"is_reliable":true}] +/// ``` +#[derive(Debug, PartialEq, Clone)] +pub struct LogsAdditionalEndpoint { + /// API key used exclusively for this endpoint. + pub api_key: String, + /// Full intake URL, e.g. `https://agent-http-intake.logs.datadoghq.com:443/api/v2/logs`. + /// Computed from `Host` and `Port` at deserialize time. + pub url: String, + /// When `true`, failures on this endpoint are counted toward overall flush reliability. + /// Currently stored but not yet acted upon; reserved for future use. + pub is_reliable: bool, +} + +/// Internal representation that mirrors the JSON wire format. +#[derive(Deserialize)] +struct RawEndpoint { + api_key: String, + #[serde(rename = "Host")] + host: String, + #[serde(rename = "Port")] + port: u32, + is_reliable: bool, +} + +impl From for LogsAdditionalEndpoint { + fn from(r: RawEndpoint) -> Self { + Self { + api_key: r.api_key, + url: format!("https://{}:{}/api/v2/logs", r.host, r.port), + is_reliable: r.is_reliable, + } + } +} + +/// Parse the value of `DD_LOGS_CONFIG_ADDITIONAL_ENDPOINTS` (a JSON array string). +/// +/// Returns an empty `Vec` and emits a warning on parse failure. +pub fn parse_additional_endpoints(s: &str) -> Vec { + match serde_json::from_str::>(s) { + Ok(raw) => raw.into_iter().map(Into::into).collect(), + Err(e) => { + warn!("failed to parse DD_LOGS_CONFIG_ADDITIONAL_ENDPOINTS: {e}"); + vec![] + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_parse_valid_endpoint() { + let s = r#"[{"api_key":"key2","Host":"agent-http-intake.logs.datadoghq.com","Port":443,"is_reliable":true}]"#; + let result = parse_additional_endpoints(s); + assert_eq!(result.len(), 1); + assert_eq!(result[0].api_key, "key2"); + assert_eq!( + result[0].url, + "https://agent-http-intake.logs.datadoghq.com:443/api/v2/logs" + ); + assert!(result[0].is_reliable); + } + + #[test] + fn test_parse_missing_port_returns_empty() { + // Missing required "Port" field — should warn and return [] + let s = r#"[{"api_key":"key","Host":"intake.logs.datadoghq.com","is_reliable":true}]"#; + let result = parse_additional_endpoints(s); + assert!(result.is_empty()); + } + + #[test] + fn test_parse_empty_string_returns_empty() { + let result = parse_additional_endpoints(""); + assert!(result.is_empty()); + } + + #[test] + fn test_parse_multiple_endpoints() { + let s = r#"[ + {"api_key":"k1","Host":"host1.example.com","Port":443,"is_reliable":true}, + {"api_key":"k2","Host":"host2.example.com","Port":10516,"is_reliable":false} + ]"#; + let result = parse_additional_endpoints(s); + assert_eq!(result.len(), 2); + assert_eq!(result[0].url, "https://host1.example.com:443/api/v2/logs"); + assert_eq!(result[1].url, "https://host2.example.com:10516/api/v2/logs"); + assert!(!result[1].is_reliable); + } +} diff --git a/crates/datadog-logs-agent/src/server.rs b/crates/datadog-logs-agent/src/server.rs new file mode 100644 index 0000000..bc7981a --- /dev/null +++ b/crates/datadog-logs-agent/src/server.rs @@ -0,0 +1,523 @@ +// Copyright 2025-Present Datadog, Inc. https://www.datadoghq.com/ +// SPDX-License-Identifier: Apache-2.0 + +//! HTTP intake server for the log agent. +//! +//! [`LogServer`] listens on a TCP port and accepts `POST /v1/input` requests +//! whose body is a JSON array of [`crate::IntakeEntry`] values. Entries are +//! forwarded to the shared [`crate::AggregatorHandle`] for batching and +//! eventual flushing. +//! +//! # Usage (network intake — serverless-compat) +//! ```ignore +//! let (service, handle) = AggregatorService::new(); +//! tokio::spawn(service.run()); +//! +//! let server = LogServer::new( +//! LogServerConfig { host: "0.0.0.0".into(), port: 8080 }, +//! handle.clone(), +//! ); +//! tokio::spawn(server.serve()); +//! +//! let flusher = LogFlusher::new(config, client, handle); +//! // flush periodically … +//! ``` +//! +//! # Direct intake (bottlecap — unchanged) +//! ```ignore +//! // bottlecap never uses LogServer; it calls handle.insert_batch() directly. +//! handle.insert_batch(entries).expect("insert"); +//! ``` + +use http_body_util::BodyExt as _; +use hyper::body::Incoming; +use hyper::service::service_fn; +use hyper::{Method, Request, Response, StatusCode}; +use hyper_util::rt::TokioIo; +use tracing::{debug, error, warn}; + +use crate::aggregator::AggregatorHandle; +use crate::intake_entry::IntakeEntry; + +const LOG_INTAKE_PATH: &str = "/v1/input"; +/// Maximum accepted request body size in bytes (4 MiB). Requests larger than +/// this are rejected with 413 before the body is read into memory. +const MAX_BODY_BYTES: usize = 4 * 1024 * 1024; + +/// Configuration for the [`LogServer`] HTTP intake listener. +#[derive(Debug, Clone)] +pub struct LogServerConfig { + /// Interface to bind (e.g. `"0.0.0.0"` or `"127.0.0.1"`). + pub host: String, + /// TCP port to listen on. + pub port: u16, +} + +/// HTTP server that receives log entries over the network and forwards them to +/// a running [`AggregatorHandle`]. +/// +/// Create with [`LogServer::new`], then call [`LogServer::serve`] inside a +/// `tokio::spawn` — it runs forever until the process exits. +pub struct LogServer { + config: LogServerConfig, + handle: AggregatorHandle, +} + +impl LogServer { + /// Create a new server. Does **not** bind the port until [`serve`](Self::serve) is called. + pub fn new(config: LogServerConfig, handle: AggregatorHandle) -> Self { + Self { config, handle } + } + + /// Bind the configured port and serve HTTP/1 requests indefinitely. + /// + /// This is an `async fn` meant to be run inside `tokio::spawn`. + /// It only returns if binding fails; otherwise it loops forever. + pub async fn serve(self) { + let addr = format!("{}:{}", self.config.host, self.config.port); + let listener = match tokio::net::TcpListener::bind(&addr).await { + Ok(l) => { + let actual = l.local_addr().map_or(addr.clone(), |a| a.to_string()); + debug!("log server listening on {actual}"); + l + } + Err(e) => { + error!("log server failed to bind {addr}: {e}"); + return; + } + }; + + loop { + let (stream, peer) = match listener.accept().await { + Ok(pair) => pair, + Err(e) => { + warn!("log server accept error: {e}"); + continue; + } + }; + + debug!("log server: connection from {peer}"); + let handle = self.handle.clone(); + tokio::spawn(async move { + let io = TokioIo::new(stream); + let svc = service_fn(move |req: Request| { + let handle = handle.clone(); + async move { handle_request(req, handle).await } + }); + if let Err(e) = hyper::server::conn::http1::Builder::new() + .serve_connection(io, svc) + .await + { + debug!("log server: connection error: {e}"); + } + }); + } + } +} + +/// Handle a single HTTP request: route, parse body, insert into aggregator. +async fn handle_request( + req: Request, + handle: AggregatorHandle, +) -> Result, std::convert::Infallible> { + if req.method() != Method::POST { + return Ok(Response::builder() + .status(StatusCode::METHOD_NOT_ALLOWED) + .body("method not allowed".to_string()) + .unwrap_or_default()); + } + if req.uri().path() != LOG_INTAKE_PATH { + return Ok(Response::builder() + .status(StatusCode::NOT_FOUND) + .body("not found".to_string()) + .unwrap_or_default()); + } + + // Reject early if Content-Length is declared and already exceeds the limit. + // Skip this check when Content-Length is absent (chunked transfer) — actual + // size is enforced after reading below. + if let Some(content_length) = req + .headers() + .get(hyper::header::CONTENT_LENGTH) + .and_then(|v| v.to_str().ok()) + .and_then(|s| s.parse::().ok()) + && content_length > MAX_BODY_BYTES + { + return Ok(Response::builder() + .status(StatusCode::PAYLOAD_TOO_LARGE) + .body("payload too large".to_string()) + .unwrap_or_default()); + } + + // TODO(SVLS-chunked-body-limit): the 4 MiB cap is only enforced *after* + // `collect()` has buffered the entire body. For chunked requests (no + // Content-Length) an attacker can stream an arbitrarily large body before + // we ever reject with 413. + // Fix: replace `req.collect()` with + // `Limited::new(req.into_body(), MAX_BODY_BYTES).collect()` from + // `http_body_util` (already a dependency). `Limited` aborts mid-stream as + // soon as the threshold is exceeded. Match on `LengthLimitError` via + // `e.downcast_ref::().is_some()` for the 413 branch, and + // remove the now-redundant post-read `bytes.len() > MAX_BODY_BYTES` check. + // Add test `test_chunked_oversized_body_returns_413`: send MAX_BODY_BYTES+1 + // bytes over raw chunked TCP using `TcpStream::into_split()` for concurrent + // write/read (needed to avoid deadlock when the OS socket buffer fills up). + let bytes = match req.collect().await { + Ok(collected) => collected.to_bytes(), + Err(e) => { + warn!("log server: failed to read request body: {e}"); + return Ok(Response::builder() + .status(StatusCode::BAD_REQUEST) + .body("failed to read body".to_string()) + .unwrap_or_default()); + } + }; + if bytes.len() > MAX_BODY_BYTES { + return Ok(Response::builder() + .status(StatusCode::PAYLOAD_TOO_LARGE) + .body("payload too large".to_string()) + .unwrap_or_default()); + } + + let entries: Vec = match serde_json::from_slice(&bytes) { + Ok(e) => e, + Err(e) => { + warn!("log server: failed to parse log entries: {e}"); + return Ok(Response::builder() + .status(StatusCode::BAD_REQUEST) + .body(format!("invalid JSON: {e}")) + .unwrap_or_default()); + } + }; + + if entries.is_empty() { + return Ok(Response::builder() + .status(StatusCode::OK) + .body("ok".to_string()) + .unwrap_or_default()); + } + + debug!("log server: received {} entries", entries.len()); + + if let Err(e) = handle.insert_batch(entries) { + error!("log server: failed to insert batch: {e}"); + return Ok(Response::builder() + .status(StatusCode::INTERNAL_SERVER_ERROR) + .body("aggregator unavailable".to_string()) + .unwrap_or_default()); + } + + Ok(Response::builder() + .status(StatusCode::OK) + .body("ok".to_string()) + .unwrap_or_default()) +} + +#[cfg(test)] +// Tests use plain reqwest; FIPS client not needed for local loopback +#[allow(clippy::disallowed_methods, clippy::unwrap_used, clippy::expect_used)] +mod tests { + use super::*; + use crate::aggregator::AggregatorService; + use tokio::time::{Duration, sleep}; + + /// Bind `:0`, record the OS-assigned port, drop the listener, then start + /// `LogServer` on that port. Returns the base URL. + async fn start_test_server(handle: AggregatorHandle) -> String { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let port = listener.local_addr().unwrap().port(); + drop(listener); + + let server = LogServer::new( + LogServerConfig { + host: "127.0.0.1".into(), + port, + }, + handle, + ); + tokio::spawn(server.serve()); + sleep(Duration::from_millis(50)).await; + format!("http://127.0.0.1:{port}") + } + + #[tokio::test] + async fn test_post_valid_entries_returns_200_and_batch_inserted() { + let (service, handle) = AggregatorService::new(); + tokio::spawn(service.run()); + + let base_url = start_test_server(handle.clone()).await; + let client = reqwest::Client::new(); + + let entries = vec![ + serde_json::json!({"message": "hello", "timestamp": 1_700_000_000_000_i64}), + serde_json::json!({"message": "world", "timestamp": 1_700_000_001_000_i64}), + ]; + + let resp = client + .post(format!("{base_url}/v1/input")) + .json(&entries) + .send() + .await + .expect("request failed"); + + assert_eq!(resp.status(), 200); + + let batches = handle.get_batches().await.expect("get_batches"); + assert_eq!(batches.len(), 1, "should have one batch"); + let arr: serde_json::Value = serde_json::from_slice(&batches[0]).unwrap(); + assert_eq!(arr.as_array().unwrap().len(), 2); + assert_eq!(arr[0]["message"], "hello"); + } + + #[tokio::test] + async fn test_post_malformed_json_returns_400() { + let (service, handle) = AggregatorService::new(); + tokio::spawn(service.run()); + + let base_url = start_test_server(handle).await; + let client = reqwest::Client::new(); + + let resp = client + .post(format!("{base_url}/v1/input")) + .header("Content-Type", "application/json") + .body("not-json") + .send() + .await + .expect("request failed"); + + assert_eq!(resp.status(), 400); + } + + #[tokio::test] + async fn test_get_request_returns_405() { + let (service, handle) = AggregatorService::new(); + tokio::spawn(service.run()); + + let base_url = start_test_server(handle).await; + let client = reqwest::Client::new(); + + let resp = client + .get(format!("{base_url}/v1/input")) + .send() + .await + .expect("request failed"); + + assert_eq!(resp.status(), 405); + } + + #[tokio::test] + async fn test_wrong_path_returns_404() { + let (service, handle) = AggregatorService::new(); + tokio::spawn(service.run()); + + let base_url = start_test_server(handle).await; + let client = reqwest::Client::new(); + + let resp = client + .post(format!("{base_url}/wrong/path")) + .json(&serde_json::json!([{"message": "x", "timestamp": 0_i64}])) + .send() + .await + .expect("request failed"); + + assert_eq!(resp.status(), 404); + } + + #[tokio::test] + async fn test_post_empty_array_returns_200_no_batch() { + let (service, handle) = AggregatorService::new(); + tokio::spawn(service.run()); + + let base_url = start_test_server(handle.clone()).await; + let client = reqwest::Client::new(); + + let resp = client + .post(format!("{base_url}/v1/input")) + .json(&serde_json::json!([])) + .send() + .await + .expect("request failed"); + + assert_eq!(resp.status(), 200); + + let batches = handle.get_batches().await.expect("get_batches"); + assert!(batches.is_empty(), "empty POST should insert nothing"); + } + + /// A request whose Content-Length header exceeds MAX_BODY_BYTES must be + /// rejected with 413 before any body bytes are read. + #[tokio::test] + async fn test_oversized_content_length_returns_413() { + let (service, handle) = AggregatorService::new(); + tokio::spawn(service.run()); + + let base_url = start_test_server(handle).await; + let client = reqwest::Client::new(); + + // The server checks the Content-Length header directly and rejects + // before reading the body when the declared size exceeds the limit. + let fake_large_size = MAX_BODY_BYTES + 1; + let resp = client + .post(format!("{base_url}/v1/input")) + .header("Content-Type", "application/json") + .header("Content-Length", fake_large_size.to_string()) + .body("[]") + .send() + .await + .expect("request failed"); + + assert_eq!(resp.status(), 413); + } + + /// A POST with Transfer-Encoding: chunked (no Content-Length header) must + /// not be rejected with 413. This is the regression test for the original + /// bug where `size_hint().upper()` returning `None` was coerced to `u64::MAX` + /// and treated as exceeding the body-size limit. + #[tokio::test] + async fn test_chunked_transfer_encoding_accepted() { + use tokio::io::{AsyncReadExt as _, AsyncWriteExt as _}; + use tokio::net::TcpStream; + + let (service, handle) = AggregatorService::new(); + tokio::spawn(service.run()); + + let base_url = start_test_server(handle.clone()).await; + let port: u16 = base_url + .trim_start_matches("http://127.0.0.1:") + .parse() + .expect("port"); + + let body = r#"[{"message":"chunked","timestamp":1700000000000}]"#; + let request = format!( + "POST /v1/input HTTP/1.1\r\n\ + Host: 127.0.0.1:{port}\r\n\ + Content-Type: application/json\r\n\ + Transfer-Encoding: chunked\r\n\ + \r\n\ + {:x}\r\n\ + {body}\r\n\ + 0\r\n\ + \r\n", + body.len(), + ); + + let mut stream = TcpStream::connect(format!("127.0.0.1:{port}")) + .await + .expect("connect"); + stream.write_all(request.as_bytes()).await.expect("write"); + stream.flush().await.expect("flush"); + + let mut response = String::new(); + let mut buf = [0u8; 4096]; + loop { + let n = stream.read(&mut buf).await.expect("read"); + if n == 0 { + break; + } + response.push_str(&String::from_utf8_lossy(&buf[..n])); + if response.contains("\r\n\r\n") { + break; + } + } + + assert!( + response.starts_with("HTTP/1.1 200"), + "expected 200, got: {response}" + ); + let batches = handle.get_batches().await.expect("get_batches"); + assert_eq!(batches.len(), 1, "entry should have been inserted"); + let arr: serde_json::Value = serde_json::from_slice(&batches[0]).unwrap(); + assert_eq!(arr[0]["message"], "chunked"); + } + + /// All optional IntakeEntry fields (hostname, service, ddsource, ddtags, + /// status) and arbitrary attributes must survive the HTTP round-trip + /// through the server and appear intact in the aggregated batch. + #[tokio::test] + async fn test_full_intake_entry_fields_preserved_through_http() { + let (service, handle) = AggregatorService::new(); + tokio::spawn(service.run()); + + let base_url = start_test_server(handle.clone()).await; + let client = reqwest::Client::new(); + + let payload = serde_json::json!([{ + "message": "lambda invoked", + "timestamp": 1_700_000_002_000_i64, + "hostname": "arn:aws:lambda:us-east-1:123:function:my-fn", + "service": "my-fn", + "ddsource": "lambda", + "ddtags": "env:prod,version:1.0", + "status": "info", + "lambda": { + "arn": "arn:aws:lambda:us-east-1:123:function:my-fn", + "request_id": "req-abc-123" + } + }]); + + let resp = client + .post(format!("{base_url}/v1/input")) + .json(&payload) + .send() + .await + .expect("request failed"); + + assert_eq!(resp.status(), 200); + + let batches = handle.get_batches().await.expect("get_batches"); + assert_eq!(batches.len(), 1); + let arr: serde_json::Value = serde_json::from_slice(&batches[0]).unwrap(); + let entry = &arr[0]; + + assert_eq!(entry["message"], "lambda invoked"); + assert_eq!( + entry["hostname"], + "arn:aws:lambda:us-east-1:123:function:my-fn" + ); + assert_eq!(entry["service"], "my-fn"); + assert_eq!(entry["ddsource"], "lambda"); + assert_eq!(entry["ddtags"], "env:prod,version:1.0"); + assert_eq!(entry["status"], "info"); + // Flattened attributes must appear at the top level + assert_eq!(entry["lambda"]["request_id"], "req-abc-123"); + } + + /// Two sequential POST requests must both accumulate in the aggregator + /// before `get_batches` drains them. + #[tokio::test] + async fn test_sequential_posts_accumulate_in_aggregator() { + let (service, handle) = AggregatorService::new(); + tokio::spawn(service.run()); + + let base_url = start_test_server(handle.clone()).await; + let client = reqwest::Client::new(); + + // First request + client + .post(format!("{base_url}/v1/input")) + .json(&serde_json::json!([{"message": "first", "timestamp": 1_i64}])) + .send() + .await + .expect("first request failed"); + + // Second request + client + .post(format!("{base_url}/v1/input")) + .json(&serde_json::json!([{"message": "second", "timestamp": 2_i64}])) + .send() + .await + .expect("second request failed"); + + let batches = handle.get_batches().await.expect("get_batches"); + // Both entries land in the same aggregator; batch count depends on + // the aggregator's internal sizing, but total entries must be 2. + let total_entries: usize = batches + .iter() + .map(|b| { + let arr: serde_json::Value = serde_json::from_slice(b).unwrap(); + arr.as_array().unwrap().len() + }) + .sum(); + assert_eq!(total_entries, 2, "both entries should be in the aggregator"); + } +} diff --git a/crates/datadog-logs-agent/tests/integration_test.rs b/crates/datadog-logs-agent/tests/integration_test.rs new file mode 100644 index 0000000..291777c --- /dev/null +++ b/crates/datadog-logs-agent/tests/integration_test.rs @@ -0,0 +1,846 @@ +// Copyright 2023-Present Datadog, Inc. https://www.datadoghq.com/ +// SPDX-License-Identifier: Apache-2.0 + +//! Integration tests for the `datadog-logs-agent` crate. +//! +//! These tests exercise two intake paths: +//! +//! **Direct intake** (bottlecap / in-process): +//! `IntakeEntry` → `AggregatorHandle::insert_batch` → `LogFlusher::flush` → HTTP endpoint +//! +//! **Network intake** (serverless-compat / over HTTP): +//! HTTP POST → `LogServer` → `AggregatorHandle::insert_batch` → `LogFlusher::flush` → HTTP endpoint +//! +//! HTTP traffic is directed to a local `mockito` server via +//! `Destination::ObservabilityPipelinesWorker`, which accepts a direct URL. +//! Datadog-mode-specific headers (`DD-PROTOCOL`) are covered by unit tests in `flusher.rs`. + +#![allow(clippy::disallowed_methods, clippy::unwrap_used, clippy::expect_used)] + +use datadog_logs_agent::{ + AggregatorService, Destination, IntakeEntry, LogFlusher, LogFlusherConfig, LogServer, + LogServerConfig, LogsAdditionalEndpoint, +}; +use mockito::{Matcher, Server}; +use std::time::Duration; +use tokio::time::sleep; + +// ── Helpers ────────────────────────────────────────────────────────────────── + +fn build_client() -> reqwest::Client { + reqwest::Client::builder() + .build() + .expect("failed to build HTTP client") +} + +/// Config that routes all flushes to `mock_url/logs` via OPW mode. +fn opw_config(mock_url: &str) -> LogFlusherConfig { + LogFlusherConfig { + api_key: "test-api-key".to_string(), + site: "ignored.datadoghq.com".to_string(), + mode: Destination::ObservabilityPipelinesWorker { + url: format!("{}/logs", mock_url), + }, + additional_endpoints: Vec::new(), + use_compression: false, + compression_level: 0, + flush_timeout: Duration::from_secs(5), + } +} + +fn entry(msg: &str) -> IntakeEntry { + IntakeEntry::from_message(msg, 1_700_000_000_000) +} + +// ── Pipeline happy path ─────────────────────────────────────────────────────── + +/// Inserting log entries and flushing sends a single POST to the endpoint. +#[tokio::test] +async fn test_pipeline_inserts_and_flushes() { + let mut server = Server::new_async().await; + let mock = server + .mock("POST", "/logs") + .match_header("DD-API-KEY", "test-api-key") + .with_status(200) + .create_async() + .await; + + let (svc, handle) = AggregatorService::new(); + let _task = tokio::spawn(svc.run()); + + handle + .insert_batch(vec![entry("hello"), entry("world")]) + .expect("insert_batch"); + + let result = LogFlusher::new(opw_config(&server.url()), build_client(), handle) + .flush(vec![]) + .await; + + assert!(result.is_empty(), "flush should return empty on 200"); + mock.assert_async().await; +} + +/// Flushing with no entries makes no HTTP request. +#[tokio::test] +async fn test_empty_flush_makes_no_request() { + let server = Server::new_async().await; + // Any unexpected request would return 501 and cause an assertion failure below. + + let (svc, handle) = AggregatorService::new(); + let _task = tokio::spawn(svc.run()); + + let url = server.url(); + let result = LogFlusher::new(opw_config(&url), build_client(), handle) + .flush(vec![]) + .await; + + assert!(result.is_empty(), "empty flush should return empty"); + // No mock was set up — if a request had been made, mockito would panic. + drop(server); +} + +// ── JSON payload shape ──────────────────────────────────────────────────────── + +/// The flushed payload is a valid JSON array containing each inserted entry. +#[tokio::test] +async fn test_payload_is_json_array_with_correct_fields() { + let (svc, handle) = AggregatorService::new(); + let _task = tokio::spawn(svc.run()); + + handle + .insert_batch(vec![IntakeEntry { + message: "user login".to_string(), + timestamp: 1_700_000_001_000, + hostname: Some("web-01".to_string()), + service: Some("auth".to_string()), + ddsource: Some("nodejs".to_string()), + ddtags: Some("env:prod,version:2.0".to_string()), + status: Some("info".to_string()), + attributes: serde_json::Map::new(), + }]) + .expect("insert"); + + let batches = handle.get_batches().await.expect("get_batches"); + assert_eq!(batches.len(), 1); + + let arr: serde_json::Value = serde_json::from_slice(&batches[0]).expect("valid JSON"); + let entries = arr.as_array().expect("JSON array"); + assert_eq!(entries.len(), 1); + + let e = &entries[0]; + assert_eq!(e["message"], "user login"); + assert_eq!(e["timestamp"], 1_700_000_001_000_i64); + assert_eq!(e["hostname"], "web-01"); + assert_eq!(e["service"], "auth"); + assert_eq!(e["ddsource"], "nodejs"); + assert_eq!(e["ddtags"], "env:prod,version:2.0"); + assert_eq!(e["status"], "info"); +} + +/// Absent optional fields are not serialized into the JSON payload. +#[tokio::test] +async fn test_absent_optional_fields_not_serialized() { + let (svc, handle) = AggregatorService::new(); + let _task = tokio::spawn(svc.run()); + + handle + .insert_batch(vec![IntakeEntry::from_message("minimal", 0)]) + .expect("insert"); + + let batches = handle.get_batches().await.expect("get_batches"); + let arr: serde_json::Value = serde_json::from_slice(&batches[0]).expect("valid JSON"); + let e = &arr[0]; + + assert_eq!(e["message"], "minimal"); + assert!(e.get("hostname").is_none(), "hostname absent"); + assert!(e.get("service").is_none(), "service absent"); + assert!(e.get("ddsource").is_none(), "ddsource absent"); + assert!(e.get("ddtags").is_none(), "ddtags absent"); + assert!(e.get("status").is_none(), "status absent"); +} + +// ── Runtime-specific attributes ─────────────────────────────────────────────── + +/// Lambda-specific attributes are flattened into the top-level JSON object. +#[tokio::test] +async fn test_lambda_attributes_flattened_at_top_level() { + let (svc, handle) = AggregatorService::new(); + let _task = tokio::spawn(svc.run()); + + let mut attrs = serde_json::Map::new(); + attrs.insert( + "lambda".to_string(), + serde_json::json!({ + "arn": "arn:aws:lambda:us-east-1:123456789012:function:my-fn", + "request_id": "abc-123" + }), + ); + + handle + .insert_batch(vec![IntakeEntry { + message: "invocation complete".to_string(), + timestamp: 0, + hostname: Some("my-fn".to_string()), + service: Some("my-fn".to_string()), + ddsource: Some("lambda".to_string()), + ddtags: Some("env:prod".to_string()), + status: Some("info".to_string()), + attributes: attrs, + }]) + .expect("insert"); + + let batches = handle.get_batches().await.expect("get_batches"); + let arr: serde_json::Value = serde_json::from_slice(&batches[0]).expect("valid JSON"); + let e = &arr[0]; + + // Lambda object is a top-level key (flattened via #[serde(flatten)]) + assert_eq!( + e["lambda"]["arn"], + "arn:aws:lambda:us-east-1:123456789012:function:my-fn" + ); + assert_eq!(e["lambda"]["request_id"], "abc-123"); +} + +/// Azure-specific attributes are flattened into the top-level JSON object. +#[tokio::test] +async fn test_azure_attributes_flattened_at_top_level() { + let (svc, handle) = AggregatorService::new(); + let _task = tokio::spawn(svc.run()); + + let mut attrs = serde_json::Map::new(); + attrs.insert( + "azure".to_string(), + serde_json::json!({ + "resource_id": "/subscriptions/sub-123/resourceGroups/rg/providers/Microsoft.Web/sites/my-fn", + "operation_name": "Microsoft.Web/sites/functions/run/action" + }), + ); + + handle + .insert_batch(vec![IntakeEntry { + message: "azure function triggered".to_string(), + timestamp: 0, + hostname: Some("my-azure-fn".to_string()), + service: Some("payments".to_string()), + ddsource: Some("azure-functions".to_string()), + ddtags: Some("env:staging".to_string()), + status: Some("info".to_string()), + attributes: attrs, + }]) + .expect("insert"); + + let batches = handle.get_batches().await.expect("get_batches"); + let arr: serde_json::Value = serde_json::from_slice(&batches[0]).expect("valid JSON"); + let e = &arr[0]; + + assert_eq!(e["ddsource"], "azure-functions"); + assert!( + e["azure"]["resource_id"] + .as_str() + .unwrap_or("") + .contains("Microsoft.Web"), + "azure resource_id present" + ); + assert_eq!( + e["azure"]["operation_name"], + "Microsoft.Web/sites/functions/run/action" + ); +} + +// ── Batch limits ────────────────────────────────────────────────────────────── + +/// Exactly MAX_BATCH_ENTRIES entries produce a single batch. +#[tokio::test] +async fn test_max_entries_fits_in_one_batch() { + const MAX: usize = datadog_logs_agent::constants::MAX_BATCH_ENTRIES; + + let (svc, handle) = AggregatorService::new(); + let _task = tokio::spawn(svc.run()); + + let entries: Vec = (0..MAX).map(|i| entry(&format!("log {i}"))).collect(); + handle.insert_batch(entries).expect("insert"); + + let batches = handle.get_batches().await.expect("get_batches"); + assert_eq!( + batches.len(), + 1, + "exactly MAX_BATCH_ENTRIES fits in one batch" + ); + + let arr: serde_json::Value = serde_json::from_slice(&batches[0]).expect("valid JSON"); + assert_eq!(arr.as_array().unwrap().len(), MAX); +} + +/// MAX_BATCH_ENTRIES + 1 entries split into two batches; two POSTs are sent. +#[tokio::test] +async fn test_overflow_produces_two_batches_and_two_posts() { + const MAX: usize = datadog_logs_agent::constants::MAX_BATCH_ENTRIES; + + let mut server = Server::new_async().await; + let mock = server + .mock("POST", "/logs") + .match_header("DD-API-KEY", "test-api-key") + .with_status(200) + .expect(2) // exactly 2 requests expected + .create_async() + .await; + + let (svc, handle) = AggregatorService::new(); + let _task = tokio::spawn(svc.run()); + + let entries: Vec = (0..=MAX).map(|i| entry(&format!("log {i}"))).collect(); + handle.insert_batch(entries).expect("insert"); + + let result = LogFlusher::new(opw_config(&server.url()), build_client(), handle) + .flush(vec![]) + .await; + + assert!(result.is_empty()); + mock.assert_async().await; +} + +// ── Oversized entries ───────────────────────────────────────────────────────── + +/// Entries exceeding MAX_LOG_BYTES are silently dropped; valid entries still flush. +#[tokio::test] +async fn test_oversized_entry_dropped_valid_entries_still_flush() { + let mut server = Server::new_async().await; + let mock = server + .mock("POST", "/logs") + .with_status(200) + .expect(1) + .create_async() + .await; + + let (svc, handle) = AggregatorService::new(); + let _task = tokio::spawn(svc.run()); + + let oversized = IntakeEntry::from_message( + "x".repeat(datadog_logs_agent::constants::MAX_LOG_BYTES + 1), + 0, + ); + let normal = entry("this one is fine"); + + handle + .insert_batch(vec![oversized, normal]) + .expect("insert"); + + let result = LogFlusher::new(opw_config(&server.url()), build_client(), handle) + .flush(vec![]) + .await; + + assert!(result.is_empty(), "flush should succeed for valid entries"); + mock.assert_async().await; +} + +/// All entries oversized means nothing to flush — no HTTP request. +#[tokio::test] +async fn test_all_oversized_entries_produces_no_request() { + let (svc, handle) = AggregatorService::new(); + let _task = tokio::spawn(svc.run()); + + let oversized = IntakeEntry::from_message( + "x".repeat(datadog_logs_agent::constants::MAX_LOG_BYTES + 1), + 0, + ); + handle.insert_batch(vec![oversized]).expect("insert"); + + let batches = handle.get_batches().await.expect("get_batches"); + assert!( + batches.is_empty(), + "oversized-only aggregator should produce no batches" + ); +} + +// ── Concurrent producers ────────────────────────────────────────────────────── + +/// Two cloned handles can insert concurrently; all entries appear in the flush. +#[tokio::test] +async fn test_concurrent_producers_all_entries_flushed() { + let mut server = Server::new_async().await; + let mock = server + .mock("POST", "/logs") + .with_status(200) + .expect(1) + .create_async() + .await; + + let (svc, handle) = AggregatorService::new(); + let _task = tokio::spawn(svc.run()); + + let h1 = handle.clone(); + let h2 = handle.clone(); + + let (r1, r2) = tokio::join!( + tokio::spawn(async move { + h1.insert_batch(vec![entry("from-producer-1")]) + .expect("h1 insert") + }), + tokio::spawn(async move { + h2.insert_batch(vec![entry("from-producer-2")]) + .expect("h2 insert") + }), + ); + r1.expect("task 1"); + r2.expect("task 2"); + + let result = LogFlusher::new(opw_config(&server.url()), build_client(), handle) + .flush(vec![]) + .await; + + assert!(result.is_empty()); + mock.assert_async().await; +} + +// ── OPW mode ────────────────────────────────────────────────────────────────── + +/// OPW mode sends to the custom URL and omits the DD-PROTOCOL header. +#[tokio::test] +async fn test_opw_mode_uses_custom_url_and_omits_dd_protocol() { + let mut server = Server::new_async().await; + let opw_path = "/opw-endpoint"; + let mock = server + .mock("POST", opw_path) + .match_header("DD-API-KEY", "test-api-key") + .match_header("DD-PROTOCOL", Matcher::Missing) + .with_status(200) + .expect(1) + .create_async() + .await; + + let (svc, handle) = AggregatorService::new(); + let _task = tokio::spawn(svc.run()); + handle.insert_batch(vec![entry("opw log")]).expect("insert"); + + let config = LogFlusherConfig { + api_key: "test-api-key".to_string(), + site: "ignored".to_string(), + mode: Destination::ObservabilityPipelinesWorker { + url: format!("{}{}", server.url(), opw_path), + }, + additional_endpoints: Vec::new(), + use_compression: false, + compression_level: 0, + flush_timeout: Duration::from_secs(5), + }; + + let result = LogFlusher::new(config, build_client(), handle) + .flush(vec![]) + .await; + + assert!(result.is_empty()); + mock.assert_async().await; +} + +// ── Compression ─────────────────────────────────────────────────────────────── + +/// OPW mode always disables compression regardless of `use_compression` setting. +/// The request must NOT carry `Content-Encoding: zstd` in OPW mode. +/// +/// Note: zstd compression in Datadog mode is verified in `flusher.rs` unit tests +/// via `ship_batch` directly, since Datadog mode constructs an HTTPS URL that +/// cannot be intercepted by a plain HTTP mock server. +#[tokio::test] +async fn test_opw_mode_disables_compression_regardless_of_config() { + let mut server = Server::new_async().await; + let mock = server + .mock("POST", "/logs") + .match_header("Content-Encoding", Matcher::Missing) // must not be compressed + .with_status(200) + .expect(1) + .create_async() + .await; + + let (svc, handle) = AggregatorService::new(); + let _task = tokio::spawn(svc.run()); + handle + .insert_batch(vec![entry("not compressed in OPW")]) + .expect("insert"); + + // use_compression: true — but OPW mode overrides this to false + let config = LogFlusherConfig { + api_key: "key".to_string(), + site: "ignored".to_string(), + mode: Destination::ObservabilityPipelinesWorker { + url: format!("{}/logs", server.url()), + }, + additional_endpoints: Vec::new(), + use_compression: true, + compression_level: 3, + flush_timeout: Duration::from_secs(5), + }; + + let result = LogFlusher::new(config, build_client(), handle) + .flush(vec![]) + .await; + + assert!(result.is_empty()); + mock.assert_async().await; +} + +// ── Retry behaviour ─────────────────────────────────────────────────────────── + +/// A transient 500 is retried; flush succeeds when the subsequent attempt returns 200. +#[tokio::test] +async fn test_retry_on_500_succeeds_on_second_attempt() { + let mut server = Server::new_async().await; + + let _fail = server + .mock("POST", "/logs") + .with_status(500) + .expect(1) + .create_async() + .await; + let _ok = server + .mock("POST", "/logs") + .with_status(200) + .expect(1) + .create_async() + .await; + + let (svc, handle) = AggregatorService::new(); + let _task = tokio::spawn(svc.run()); + handle + .insert_batch(vec![entry("retry me")]) + .expect("insert"); + + let result = LogFlusher::new(opw_config(&server.url()), build_client(), handle) + .flush(vec![]) + .await; + + assert!(result.is_empty(), "should succeed after retry"); +} + +/// A 403 is a permanent error; flush fails without additional retry attempts. +#[tokio::test] +async fn test_permanent_error_on_403_no_retry() { + let mut server = Server::new_async().await; + let mock = server + .mock("POST", "/logs") + .with_status(403) + .expect(1) // must be called exactly once — no retries + .create_async() + .await; + + let (svc, handle) = AggregatorService::new(); + let _task = tokio::spawn(svc.run()); + handle + .insert_batch(vec![entry("forbidden")]) + .expect("insert"); + + let result = LogFlusher::new(opw_config(&server.url()), build_client(), handle) + .flush(vec![]) + .await; + + // 403 is a permanent error — dropped silently; no builder to retry. + assert!( + result.is_empty(), + "403 is a permanent error; no retry builder returned" + ); + mock.assert_async().await; +} + +/// All three retry attempts fail with 503; flush returns false. +#[tokio::test] +async fn test_exhausted_retries_returns_false() { + let mut server = Server::new_async().await; + let mock = server + .mock("POST", "/logs") + .with_status(503) + .expect(3) // MAX_FLUSH_ATTEMPTS = 3 + .create_async() + .await; + + let (svc, handle) = AggregatorService::new(); + let _task = tokio::spawn(svc.run()); + handle + .insert_batch(vec![entry("keep failing")]) + .expect("insert"); + + let result = LogFlusher::new(opw_config(&server.url()), build_client(), handle) + .flush(vec![]) + .await; + + // Transient 503 exhausts per-invocation retries; builder returned for next flush. + assert!( + !result.is_empty(), + "exhausted retries should return a retry builder" + ); + mock.assert_async().await; +} + +// ── Additional endpoints ────────────────────────────────────────────────────── + +/// When additional endpoints are configured, the same batch is shipped to each. +#[tokio::test] +async fn test_additional_endpoints_receive_same_batch() { + let mut primary = Server::new_async().await; + let mut secondary = Server::new_async().await; + + let primary_mock = primary + .mock("POST", "/logs") + .with_status(200) + .expect(1) + .create_async() + .await; + + let secondary_mock = secondary + .mock("POST", "/extra") + .with_status(200) + .expect(1) + .create_async() + .await; + + let (svc, handle) = AggregatorService::new(); + let _task = tokio::spawn(svc.run()); + handle + .insert_batch(vec![entry("multi-endpoint")]) + .expect("insert"); + + let config = LogFlusherConfig { + api_key: "test-api-key".to_string(), + site: "ignored".to_string(), + mode: Destination::ObservabilityPipelinesWorker { + url: format!("{}/logs", primary.url()), + }, + additional_endpoints: vec![LogsAdditionalEndpoint { + api_key: "secondary-api-key".to_string(), + url: format!("{}/extra", secondary.url()), + is_reliable: true, + }], + use_compression: false, + compression_level: 0, + flush_timeout: Duration::from_secs(5), + }; + + let result = LogFlusher::new(config, build_client(), handle) + .flush(vec![]) + .await; + + assert!(result.is_empty()); + primary_mock.assert_async().await; + secondary_mock.assert_async().await; +} + +/// Additional endpoint failure does not cause flush() to return false +/// (additional endpoints are best-effort). +#[tokio::test] +async fn test_additional_endpoint_failure_does_not_affect_return_value() { + let mut primary = Server::new_async().await; + let mut secondary = Server::new_async().await; + + let _primary_mock = primary + .mock("POST", "/logs") + .with_status(200) + .create_async() + .await; + + let _secondary_mock = secondary + .mock("POST", "/extra") + .with_status(500) // secondary always fails + .create_async() + .await; + + let (svc, handle) = AggregatorService::new(); + let _task = tokio::spawn(svc.run()); + handle.insert_batch(vec![entry("test")]).expect("insert"); + + let config = LogFlusherConfig { + api_key: "key".to_string(), + site: "ignored".to_string(), + mode: Destination::ObservabilityPipelinesWorker { + url: format!("{}/logs", primary.url()), + }, + additional_endpoints: vec![LogsAdditionalEndpoint { + api_key: "secondary-api-key".to_string(), + url: format!("{}/extra", secondary.url()), + is_reliable: true, + }], + use_compression: false, + compression_level: 0, + flush_timeout: Duration::from_secs(5), + }; + + let result = LogFlusher::new(config, build_client(), handle) + .flush(vec![]) + .await; + + assert!( + result.is_empty(), + "primary succeeded — additional endpoint failure must not affect return value" + ); +} + +// ── Network intake (LogServer) ──────────────────────────────────────────────── + +/// Bind :0 to get a free port, drop the listener, then start LogServer on that +/// port. Returns the base URL ("http://127.0.0.1:"). +async fn start_log_server(handle: datadog_logs_agent::AggregatorHandle) -> String { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0") + .await + .expect("bind :0"); + let port = listener.local_addr().expect("local_addr").port(); + drop(listener); + + let server = LogServer::new( + LogServerConfig { + host: "127.0.0.1".into(), + port, + }, + handle, + ); + tokio::spawn(server.serve()); + sleep(Duration::from_millis(50)).await; // allow server to bind + format!("http://127.0.0.1:{port}") +} + +/// Full network-intake pipeline: HTTP POST → LogServer → AggregatorService → +/// LogFlusher → mockito backend. This mirrors what serverless-compat does +/// when DD_LOGS_ENABLED=true. +#[tokio::test] +async fn test_server_to_flusher_full_pipeline() { + let mut backend = Server::new_async().await; + let mock = backend + .mock("POST", "/logs") + .match_header("DD-API-KEY", "test-api-key") + .with_status(200) + .expect(1) + .create_async() + .await; + + let (svc, handle) = AggregatorService::new(); + tokio::spawn(svc.run()); + + let base_url = start_log_server(handle.clone()).await; + + // External adapter POSTs a log entry over HTTP (as serverless-compat extension would). + let client = reqwest::Client::new(); + let resp = client + .post(format!("{base_url}/v1/input")) + .json(&serde_json::json!([{ + "message": "invocation start", + "timestamp": 1_700_000_000_000_i64, + "ddsource": "lambda", + "service": "my-fn", + "ddtags": "env:prod" + }])) + .send() + .await + .expect("POST to log server"); + + assert_eq!(resp.status(), 200, "log server should accept the entry"); + + // Flush everything accumulated in the aggregator to the mock backend. + let result = LogFlusher::new(opw_config(&backend.url()), build_client(), handle) + .flush(vec![]) + .await; + + assert!(result.is_empty(), "flush should return empty on 200"); + mock.assert_async().await; +} + +/// Multiple concurrent HTTP clients can POST entries simultaneously; all +/// entries must arrive in the aggregator before flushing. +#[tokio::test] +async fn test_server_concurrent_clients_all_entries_arrive() { + let mut backend = Server::new_async().await; + let mock = backend + .mock("POST", "/logs") + .with_status(200) + .expect(1) + .create_async() + .await; + + let (svc, handle) = AggregatorService::new(); + tokio::spawn(svc.run()); + + let base_url = start_log_server(handle.clone()).await; + + // Five concurrent producers each POST one entry. + const N: usize = 5; + let mut tasks = Vec::with_capacity(N); + for i in 0..N { + let url = format!("{base_url}/v1/input"); + tasks.push(tokio::spawn(async move { + reqwest::Client::new() + .post(&url) + .json(&serde_json::json!([{ + "message": format!("entry-{i}"), + "timestamp": i as i64 + }])) + .send() + .await + .expect("concurrent POST") + .status() + })); + } + + for task in tasks { + let status = task.await.expect("task"); + assert_eq!(status, 200); + } + + // All N entries must be present in the aggregator. + let batches = handle.get_batches().await.expect("get_batches"); + let total: usize = batches + .iter() + .map(|b| { + let arr: serde_json::Value = serde_json::from_slice(b).unwrap(); + arr.as_array().unwrap().len() + }) + .sum(); + assert_eq!(total, N, "all {N} concurrent entries must be aggregated"); + + // Re-insert the drained entries so we have something to flush. + let (svc2, handle2) = AggregatorService::new(); + tokio::spawn(svc2.run()); + handle2 + .insert_batch(vec![entry("placeholder")]) + .expect("insert"); + let result = LogFlusher::new(opw_config(&backend.url()), build_client(), handle2) + .flush(vec![]) + .await; + assert!(result.is_empty()); + mock.assert_async().await; +} + +/// A malformed POST (invalid JSON) must return 400 and must not prevent the +/// server from processing subsequent valid requests. +#[tokio::test] +async fn test_server_invalid_request_does_not_block_subsequent_valid_requests() { + let (svc, handle) = AggregatorService::new(); + tokio::spawn(svc.run()); + + let base_url = start_log_server(handle.clone()).await; + let client = reqwest::Client::new(); + let url = format!("{base_url}/v1/input"); + + // Bad JSON → 400 + let bad = client + .post(&url) + .header("Content-Type", "application/json") + .body("not-json-at-all") + .send() + .await + .expect("bad POST"); + assert_eq!(bad.status(), 400); + + // Valid entry immediately after → 200 and entry reaches aggregator + let good = client + .post(&url) + .json(&serde_json::json!([{"message": "after-error", "timestamp": 1_i64}])) + .send() + .await + .expect("good POST"); + assert_eq!(good.status(), 200); + + let batches = handle.get_batches().await.expect("get_batches"); + let total: usize = batches + .iter() + .map(|b| { + let arr: serde_json::Value = serde_json::from_slice(b).unwrap(); + arr.as_array().unwrap().len() + }) + .sum(); + assert_eq!(total, 1, "only the valid entry should be in the aggregator"); +} diff --git a/crates/datadog-serverless-compat/Cargo.toml b/crates/datadog-serverless-compat/Cargo.toml index ed57366..6d99940 100644 --- a/crates/datadog-serverless-compat/Cargo.toml +++ b/crates/datadog-serverless-compat/Cargo.toml @@ -10,8 +10,9 @@ default = [] windows-pipes = ["datadog-trace-agent/windows-pipes", "dogstatsd/windows-pipes"] [dependencies] +datadog-logs-agent = { path = "../datadog-logs-agent" } datadog-trace-agent = { path = "../datadog-trace-agent" } -libdd-trace-utils = { git = "https://github.com/DataDog/libdatadog", rev = "d52ee90209cb12a28bdda0114535c1a985a29d95" } +libdd-trace-utils = { git = "https://github.com/DataDog/libdatadog", rev = "986aab55cb7941d8453dffb59d35a70599d08665" } datadog-fips = { path = "../datadog-fips", default-features = false } dogstatsd = { path = "../dogstatsd", default-features = true } reqwest = { version = "0.12.4", default-features = false } @@ -27,5 +28,10 @@ tracing-subscriber = { version = "0.3", default-features = false, features = [ ] } zstd = { version = "0.13.3", default-features = false } +[dev-dependencies] +reqwest = { version = "0.12.4", features = ["json"], default-features = false } +serde_json = { version = "1.0.116", default-features = false, features = ["alloc"] } +tokio = { version = "1", features = ["macros", "rt-multi-thread", "time"] } + [[bin]] name = "datadog-serverless-compat" diff --git a/crates/datadog-serverless-compat/src/main.rs b/crates/datadog-serverless-compat/src/main.rs index d50798f..8c20c41 100644 --- a/crates/datadog-serverless-compat/src/main.rs +++ b/crates/datadog-serverless-compat/src/main.rs @@ -12,7 +12,7 @@ use tokio::{ sync::Mutex as TokioMutex, time::{Duration, interval}, }; -use tracing::{debug, error, info}; +use tracing::{debug, error, info, warn}; use tracing_subscriber::EnvFilter; use zstd::zstd_safe::CompressionLevel; @@ -26,6 +26,10 @@ use datadog_trace_agent::{ use libdd_trace_utils::{config_utils::read_cloud_env, trace_utils::EnvironmentType}; use datadog_fips::reqwest_adapter::create_reqwest_client_builder; +use datadog_logs_agent::{ + AggregatorHandle as LogAggregatorHandle, AggregatorService as LogAggregatorService, + Destination as LogDestination, LogFlusher, LogFlusherConfig, LogServer, LogServerConfig, +}; use dogstatsd::{ aggregator::{AggregatorHandle, AggregatorService}, api_key::ApiKeyFactory, @@ -42,6 +46,7 @@ use tokio_util::sync::CancellationToken; const DOGSTATSD_FLUSH_INTERVAL: u64 = 10; const DOGSTATSD_TIMEOUT_DURATION: Duration = Duration::from_secs(5); const DEFAULT_DOGSTATSD_PORT: u16 = 8125; +const DEFAULT_LOG_INTAKE_PORT: u16 = 10517; const AGENT_HOST: &str = "0.0.0.0"; #[tokio::main] @@ -107,6 +112,13 @@ pub async fn main() { let https_proxy = env::var("DD_PROXY_HTTPS") .or_else(|_| env::var("HTTPS_PROXY")) .ok(); + let dd_logs_enabled = env::var("DD_LOGS_ENABLED") + .map(|val| val.to_lowercase() == "true") + .unwrap_or(false); + let dd_logs_port: u16 = env::var("DD_LOGS_PORT") + .ok() + .and_then(|v| v.parse::().ok()) + .unwrap_or(DEFAULT_LOG_INTAKE_PORT); debug!("Starting serverless trace mini agent"); let env_filter = format!("h2=off,hyper=off,rustls=off,{}", log_level); @@ -174,9 +186,9 @@ pub async fn main() { debug!("Starting dogstatsd"); let (_, metrics_flusher, aggregator_handle) = start_dogstatsd( dd_dogstatsd_port, - dd_api_key, + dd_api_key.clone(), dd_site, - https_proxy, + https_proxy.clone(), dogstatsd_tags, dd_statsd_metric_namespace, #[cfg(all(windows, feature = "windows-pipes"))] @@ -194,9 +206,31 @@ pub async fn main() { (None, None) }; + let (log_flusher, _log_aggregator_handle): (Option, Option) = + if dd_logs_enabled { + debug!("Starting log agent"); + match start_log_agent(dd_api_key, https_proxy, dd_logs_port) { + Some((flusher, handle)) => { + info!("log agent started"); + (Some(flusher), Some(handle)) + } + None => { + warn!("log agent failed to start, log flushing disabled"); + (None, None) + } + } + } else { + info!("log agent disabled"); + (None, None) + }; + let mut flush_interval = interval(Duration::from_secs(DOGSTATSD_FLUSH_INTERVAL)); flush_interval.tick().await; // discard first tick, which is instantaneous + // Builders for log batches that failed transiently in the previous flush + // cycle. They are redriven on the next cycle before new batches are sent. + let mut pending_log_retries: Vec = Vec::new(); + loop { flush_interval.tick().await; @@ -204,6 +238,22 @@ pub async fn main() { debug!("Flushing dogstatsd metrics"); metrics_flusher.flush().await; } + + if let Some(log_flusher) = log_flusher.as_ref() { + debug!("Flushing log agent"); + let retry_in = std::mem::take(&mut pending_log_retries); + let failed = log_flusher.flush(retry_in).await; + if !failed.is_empty() { + // TODO: surface flush failures into health/metrics telemetry so + // operators have a durable signal beyond log lines when logs are + // being dropped (e.g. increment a statsd counter or set a gauge). + warn!( + "log agent flush failed for {} batch(es); will retry next cycle", + failed.len() + ); + pending_log_retries = failed; + } + } } } @@ -312,3 +362,187 @@ fn build_metrics_client( } Ok(builder.build()?) } + +fn start_log_agent( + dd_api_key: Option, + https_proxy: Option, + logs_port: u16, +) -> Option<(LogFlusher, LogAggregatorHandle)> { + let Some(api_key) = dd_api_key else { + error!("DD_API_KEY not set, log agent disabled"); + return None; + }; + + let (service, handle): (LogAggregatorService, LogAggregatorHandle) = + LogAggregatorService::new(); + tokio::spawn(service.run()); + + let client = create_reqwest_client_builder() + .map_err(|e| error!("failed to create FIPS HTTP client for log agent: {e}")) + .ok() + .and_then(|b| { + let mut builder = b.timeout(DOGSTATSD_TIMEOUT_DURATION); + if let Some(ref proxy) = https_proxy { + match reqwest::Proxy::https(proxy.as_str()) { + Ok(p) => builder = builder.proxy(p), + Err(e) => error!("invalid HTTPS proxy for log agent: {e}"), + } + } + match builder.build() { + Ok(c) => Some(c), + Err(e) => { + error!("failed to build HTTP client for log agent: {e}"); + None + } + } + }); + + let client = client?; // error already logged above + + let config = LogFlusherConfig { + api_key, + ..LogFlusherConfig::from_env() + }; + + // Fail fast: OPW mode with an empty URL will always produce a network error at flush time. + if let LogDestination::ObservabilityPipelinesWorker { url } = &config.mode + && url.is_empty() + { + error!( + "OPW mode enabled but DD_OBSERVABILITY_PIPELINES_WORKER_LOGS_URL is empty — log agent disabled" + ); + return None; + } + + // Start the HTTP intake server so external adapters can POST log entries. + let server = LogServer::new( + LogServerConfig { + host: AGENT_HOST.to_string(), + port: logs_port, + }, + handle.clone(), + ); + // TODO(SVLS-bind-fail-fast): `LogServer::serve` binds the port inside the + // spawned task, so any bind failure (e.g. port already in use) is only + // logged as an error and silently swallowed — this function still returns + // `Some(...)` and the caller logs "log agent started" even though the + // server never came up. + // Fix: split `LogServer` into a `bind() -> Result` + // step and a `BoundLogServer::serve()` accept-loop step (both in server.rs). + // Make this fn `async`, call `server.bind().await`, return `None` on error, + // and only spawn `bound.serve()` after a successful bind. Add tests: + // `test_bind_returns_err_when_port_already_in_use` (server.rs) and + // `test_start_log_agent_returns_none_when_port_already_in_use` (main.rs). + tokio::spawn(server.serve()); + info!("log server listening on {AGENT_HOST}:{logs_port}"); + + let flusher = LogFlusher::new(config, client, handle.clone()); + Some((flusher, handle)) +} + +#[cfg(test)] +mod log_agent_integration_tests { + use datadog_logs_agent::{AggregatorService, IntakeEntry, LogServer, LogServerConfig}; + + #[tokio::test] + async fn test_log_agent_full_pipeline_compiles_and_runs() { + let (service, handle) = AggregatorService::new(); + tokio::spawn(service.run()); + + handle + .insert_batch(vec![IntakeEntry { + message: "azure function invoked".to_string(), + timestamp: 1_700_000_000_000, + hostname: Some("my-azure-fn".to_string()), + service: Some("payments".to_string()), + ddsource: Some("azure-functions".to_string()), + ddtags: Some("env:prod".to_string()), + status: Some("info".to_string()), + attributes: serde_json::Map::new(), + }]) + .expect("insert_batch"); + + let batches = handle.get_batches().await.expect("get_batches"); + assert_eq!(batches.len(), 1); + + let arr: serde_json::Value = serde_json::from_slice(&batches[0]).expect("json"); + assert_eq!(arr[0]["ddsource"], "azure-functions"); + assert_eq!(arr[0]["service"], "payments"); + + handle.shutdown().expect("shutdown"); + } + + /// start_log_agent must return None when OPW mode is enabled but the URL is empty. + #[tokio::test] + async fn test_opw_empty_url_is_detected() { + use super::start_log_agent; + // Enable OPW mode with a deliberately empty URL — the production guard + // inside start_log_agent must catch this and return None. + // SAFETY: test-only, single-threaded setup before any spawned tasks. + unsafe { + std::env::set_var("DD_OBSERVABILITY_PIPELINES_WORKER_LOGS_ENABLED", "true"); + std::env::set_var("DD_OBSERVABILITY_PIPELINES_WORKER_LOGS_URL", ""); + } + let result = start_log_agent(Some("test-key".to_string()), None, 0); + // SAFETY: test-only cleanup. + unsafe { + std::env::remove_var("DD_OBSERVABILITY_PIPELINES_WORKER_LOGS_ENABLED"); + std::env::remove_var("DD_OBSERVABILITY_PIPELINES_WORKER_LOGS_URL"); + } + assert!( + result.is_none(), + "start_log_agent must return None when OPW URL is empty" + ); + } + + /// Full network intake path: entries posted over HTTP to LogServer must + /// reach the AggregatorService and be retrievable via get_batches. + /// This mirrors what serverless-compat does when DD_LOGS_ENABLED=true. + #[tokio::test] + #[allow(clippy::disallowed_methods, clippy::unwrap_used, clippy::expect_used)] + async fn test_log_server_network_intake_end_to_end() { + use tokio::time::{Duration, sleep}; + + let (service, handle) = AggregatorService::new(); + tokio::spawn(service.run()); + + // Bind :0 to discover a free port, then hand it to LogServer + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let port = listener.local_addr().unwrap().port(); + drop(listener); + + let server = LogServer::new( + LogServerConfig { + host: "127.0.0.1".into(), + port, + }, + handle.clone(), + ); + tokio::spawn(server.serve()); + sleep(Duration::from_millis(50)).await; + + let client = reqwest::Client::new(); + let resp = client + .post(format!("http://127.0.0.1:{port}/v1/input")) + .json(&serde_json::json!([{ + "message": "lambda function invoked", + "timestamp": 1_700_000_000_000_i64, + "ddsource": "lambda", + "service": "my-fn" + }])) + .send() + .await + .expect("POST to log server failed"); + + assert_eq!(resp.status(), 200, "server must accept the payload"); + + let batches = handle.get_batches().await.expect("get_batches"); + assert_eq!(batches.len(), 1, "one batch expected"); + let arr: serde_json::Value = serde_json::from_slice(&batches[0]).expect("json"); + assert_eq!(arr[0]["message"], "lambda function invoked"); + assert_eq!(arr[0]["ddsource"], "lambda"); + assert_eq!(arr[0]["service"], "my-fn"); + + handle.shutdown().expect("shutdown"); + } +} diff --git a/crates/datadog-trace-agent/Cargo.toml b/crates/datadog-trace-agent/Cargo.toml index 4cc5ed7..d057984 100644 --- a/crates/datadog-trace-agent/Cargo.toml +++ b/crates/datadog-trace-agent/Cargo.toml @@ -24,12 +24,13 @@ async-trait = "0.1.64" tracing = { version = "0.1", default-features = false } serde = { version = "1.0.145", features = ["derive"] } serde_json = "1.0" -libdd-common = { git = "https://github.com/DataDog/libdatadog", rev = "d52ee90209cb12a28bdda0114535c1a985a29d95" } -libdd-trace-protobuf = { git = "https://github.com/DataDog/libdatadog", rev = "d52ee90209cb12a28bdda0114535c1a985a29d95" } -libdd-trace-utils = { git = "https://github.com/DataDog/libdatadog", rev = "d52ee90209cb12a28bdda0114535c1a985a29d95", features = [ +libdd-capabilities = { git = "https://github.com/DataDog/libdatadog", rev = "986aab55cb7941d8453dffb59d35a70599d08665" } +libdd-common = { git = "https://github.com/DataDog/libdatadog", rev = "950562bb191205bf16edbb4296e4a8ae33da194c" } +libdd-trace-protobuf = { git = "https://github.com/DataDog/libdatadog", rev = "986aab55cb7941d8453dffb59d35a70599d08665" } +libdd-trace-utils = { git = "https://github.com/DataDog/libdatadog", rev = "986aab55cb7941d8453dffb59d35a70599d08665", features = [ "mini_agent", ] } -libdd-trace-obfuscation = { git = "https://github.com/DataDog/libdatadog", rev = "d52ee90209cb12a28bdda0114535c1a985a29d95" } +libdd-trace-obfuscation = { git = "https://github.com/DataDog/libdatadog", rev = "986aab55cb7941d8453dffb59d35a70599d08665" } datadog-fips = { path = "../datadog-fips" } reqwest = { version = "0.12.23", features = ["json", "http2"], default-features = false } bytes = "1.10.1" @@ -37,9 +38,9 @@ bytes = "1.10.1" [dev-dependencies] rmp-serde = "1.1.1" serial_test = "2.0.0" -duplicate = "0.4.1" +duplicate = "2.0.1" temp-env = "0.3.6" tempfile = "3.3.0" -libdd-trace-utils = { git = "https://github.com/DataDog/libdatadog", rev = "d52ee90209cb12a28bdda0114535c1a985a29d95", features = [ +libdd-trace-utils = { git = "https://github.com/DataDog/libdatadog", rev = "986aab55cb7941d8453dffb59d35a70599d08665", features = [ "test-utils", ] } diff --git a/crates/datadog-trace-agent/src/aggregator.rs b/crates/datadog-trace-agent/src/aggregator.rs index 4f36424..b7467ee 100644 --- a/crates/datadog-trace-agent/src/aggregator.rs +++ b/crates/datadog-trace-agent/src/aggregator.rs @@ -104,7 +104,7 @@ mod tests { aggregator.add(payload); assert_eq!(aggregator.queue.len(), 1); - assert_eq!(aggregator.queue[0].is_empty(), false); + assert!(!aggregator.queue[0].is_empty()); assert_eq!(aggregator.queue[0].len(), 1); } diff --git a/crates/datadog-trace-agent/src/config.rs b/crates/datadog-trace-agent/src/config.rs index 5a7b8a8..ddbd820 100644 --- a/crates/datadog-trace-agent/src/config.rs +++ b/crates/datadog-trace-agent/src/config.rs @@ -296,6 +296,7 @@ mod tests { [ ("DD_API_KEY", Some("_not_a_real_key_")), ("K_SERVICE", Some("function_name")), + ("FUNCTION_TARGET", Some("function_target")), ], || { let config_res = config::Config::new(); @@ -329,6 +330,7 @@ mod tests { [ ("DD_API_KEY", Some("_not_a_real_key_")), ("K_SERVICE", Some("function_name")), + ("FUNCTION_TARGET", Some("function_target")), ("DD_SITE", Some(dd_site)), ], || { @@ -356,6 +358,7 @@ mod tests { [ ("DD_API_KEY", Some("_not_a_real_key_")), ("K_SERVICE", Some("function_name")), + ("FUNCTION_TARGET", Some("function_target")), ("DD_SITE", Some(dd_site)), ], || { @@ -374,6 +377,7 @@ mod tests { [ ("DD_API_KEY", Some("_not_a_real_key_")), ("K_SERVICE", Some("function_name")), + ("FUNCTION_TARGET", Some("function_target")), ("DD_APM_DD_URL", Some("http://127.0.0.1:3333")), ], || { diff --git a/crates/datadog-trace-agent/src/env_verifier.rs b/crates/datadog-trace-agent/src/env_verifier.rs index 29fcc5c..9ece700 100644 --- a/crates/datadog-trace-agent/src/env_verifier.rs +++ b/crates/datadog-trace-agent/src/env_verifier.rs @@ -4,7 +4,7 @@ use async_trait::async_trait; use http_body_util::BodyExt; use hyper::{Method, Request}; -use libdd_common::hyper_migration; +use libdd_common::http_common; use serde::{Deserialize, Serialize}; use std::env; use std::fs; @@ -186,24 +186,24 @@ fn get_region_from_gcp_region_string(str: String) -> String { /// tests #[async_trait] pub(crate) trait GoogleMetadataClient { - async fn get_metadata(&self) -> anyhow::Result; + async fn get_metadata(&self) -> anyhow::Result; } struct GoogleMetadataClientWrapper {} #[async_trait] impl GoogleMetadataClient for GoogleMetadataClientWrapper { - async fn get_metadata(&self) -> anyhow::Result { + async fn get_metadata(&self) -> anyhow::Result { let req = Request::builder() .method(Method::POST) .uri(GCP_METADATA_URL) .header("Metadata-Flavor", "Google") - .body(hyper_migration::Body::empty()) + .body(http_common::Body::empty()) .map_err(|err| anyhow::anyhow!(err.to_string()))?; - let client = hyper_migration::new_default_client(); + let client = http_common::new_default_client(); match client.request(req).await { - Ok(res) => Ok(hyper_migration::into_response(res)), + Ok(res) => Ok(http_common::into_response(res)), Err(err) => anyhow::bail!(err.to_string()), } } @@ -243,7 +243,7 @@ async fn ensure_gcp_function_environment( Ok(gcp_metadata) } -async fn get_gcp_metadata_from_body(body: hyper_migration::Body) -> anyhow::Result { +async fn get_gcp_metadata_from_body(body: http_common::Body) -> anyhow::Result { let bytes = body.collect().await?.to_bytes(); let body_str = String::from_utf8(bytes.to_vec())?; let gcp_metadata: GCPMetadata = serde_json::from_str(&body_str)?; @@ -361,7 +361,7 @@ async fn ensure_azure_function_environment( mod tests { use async_trait::async_trait; use hyper::{Response, StatusCode, body::Bytes}; - use libdd_common::hyper_migration; + use libdd_common::http_common; use libdd_trace_utils::trace_utils; use serde_json::json; use serial_test::serial; @@ -382,7 +382,7 @@ mod tests { struct MockGoogleMetadataClient {} #[async_trait] impl GoogleMetadataClient for MockGoogleMetadataClient { - async fn get_metadata(&self) -> anyhow::Result { + async fn get_metadata(&self) -> anyhow::Result { anyhow::bail!("Random Error") } } @@ -401,9 +401,9 @@ mod tests { struct MockGoogleMetadataClient {} #[async_trait] impl GoogleMetadataClient for MockGoogleMetadataClient { - async fn get_metadata(&self) -> anyhow::Result { + async fn get_metadata(&self) -> anyhow::Result { Ok( - hyper_migration::empty_response(Response::builder().status(StatusCode::OK)) + http_common::empty_response(Response::builder().status(StatusCode::OK)) .unwrap(), ) } @@ -423,8 +423,8 @@ mod tests { struct MockGoogleMetadataClient {} #[async_trait] impl GoogleMetadataClient for MockGoogleMetadataClient { - async fn get_metadata(&self) -> anyhow::Result { - Ok(hyper_migration::empty_response( + async fn get_metadata(&self) -> anyhow::Result { + Ok(http_common::empty_response( Response::builder() .status(StatusCode::OK) .header("Server", "Metadata Server NOT for Serverless"), @@ -447,8 +447,8 @@ mod tests { struct MockGoogleMetadataClient {} #[async_trait] impl GoogleMetadataClient for MockGoogleMetadataClient { - async fn get_metadata(&self) -> anyhow::Result { - Ok(hyper_migration::mock_response( + async fn get_metadata(&self) -> anyhow::Result { + Ok(http_common::mock_response( Response::builder() .status(StatusCode::OK) .header("Server", "Metadata Server for Serverless"), @@ -489,11 +489,11 @@ mod tests { struct MockGoogleMetadataClient {} #[async_trait] impl GoogleMetadataClient for MockGoogleMetadataClient { - async fn get_metadata(&self) -> anyhow::Result { + async fn get_metadata(&self) -> anyhow::Result { // Sleep for 5 seconds to let the timeout trigger tokio::time::sleep(Duration::from_secs(5)).await; Ok( - hyper_migration::empty_response(Response::builder().status(StatusCode::OK)) + http_common::empty_response(Response::builder().status(StatusCode::OK)) .unwrap(), ) } diff --git a/crates/datadog-trace-agent/src/http_utils.rs b/crates/datadog-trace-agent/src/http_utils.rs index 74bc510..6061566 100644 --- a/crates/datadog-trace-agent/src/http_utils.rs +++ b/crates/datadog-trace-agent/src/http_utils.rs @@ -7,7 +7,7 @@ use hyper::{ Response, StatusCode, header, http::{self, HeaderMap}, }; -use libdd_common::hyper_migration; +use libdd_common::http_common; use serde_json::json; use std::error::Error; use tracing::{debug, error}; @@ -24,7 +24,7 @@ use tracing::{debug, error}; pub fn log_and_create_http_response( message: &str, status: StatusCode, -) -> http::Result> { +) -> http::Result> { if status.is_success() { debug!("{message}"); } else { @@ -33,7 +33,7 @@ pub fn log_and_create_http_response( let body = json!({ "message": message }).to_string(); Response::builder() .status(status) - .body(hyper_migration::Body::from(body)) + .body(http_common::Body::from(body)) } /// Does two things: @@ -50,12 +50,12 @@ pub fn log_and_create_http_response( pub fn log_and_create_traces_success_http_response( message: &str, status: StatusCode, -) -> http::Result { +) -> http::Result { debug!("{message}"); let body = json!({"rate_by_service":{"service:,env:":1}}).to_string(); Response::builder() .status(status) - .body(hyper_migration::Body::from(body)) + .body(http_common::Body::from(body)) } /// Takes a request's header map, and verifies that the "content-length" and/or "Transfer-Encoding" header @@ -67,7 +67,7 @@ pub fn verify_request_content_length( header_map: &HeaderMap, max_content_length: usize, error_message_prefix: &str, -) -> Option> { +) -> Option> { let content_length_header = match header_map.get(header::CONTENT_LENGTH) { Some(res) => res, None => { @@ -113,6 +113,25 @@ pub fn verify_request_content_length( None } +/// Environment variable set by the Lambda runtime to indicate the initialisation type. +/// Lambda Web Adapter (web function / native-http mode) sets this to `"native-http"`; +/// standard on-demand invocations set it to `"on-demand"`. +const ENV_LAMBDA_INIT_TYPE: &str = "AWS_LAMBDA_INITIALIZATION_TYPE"; + +/// Returns true if the current environment is Lambda Lite (web function / native-http mode). +/// +/// Determined by checking [`ENV_LAMBDA_INIT_TYPE`] == `"native-http"`. This is used to gate +/// behaviour specific to long-running web server deployments on Lambda Lite. +pub fn is_lambda_lite() -> bool { + is_lambda_lite_from_env(std::env::var(ENV_LAMBDA_INIT_TYPE).ok().as_deref()) +} + +// Split out from `is_lambda_lite` for testability — allows injecting the env value in unit tests +// without mutating process-global state. +fn is_lambda_lite_from_env(val: Option<&str>) -> bool { + val == Some("native-http") +} + /// Builds a reqwest client with optional proxy configuration and timeout. /// Uses rustls TLS by default. FIPS-compliant TLS is available via the fips feature pub fn build_client( @@ -132,17 +151,38 @@ mod tests { use hyper::HeaderMap; use hyper::StatusCode; use hyper::header; - use libdd_common::hyper_migration; + use libdd_common::http_common; + use super::is_lambda_lite_from_env; use super::verify_request_content_length; + #[test] + fn test_is_lambda_lite_native_http() { + assert!(is_lambda_lite_from_env(Some("native-http"))); + } + + #[test] + fn test_is_lambda_lite_on_demand() { + assert!(!is_lambda_lite_from_env(Some("on-demand"))); + } + + #[test] + fn test_is_lambda_lite_empty_string() { + assert!(!is_lambda_lite_from_env(Some(""))); + } + + #[test] + fn test_is_lambda_lite_unset() { + assert!(!is_lambda_lite_from_env(None)); + } + fn create_test_headers_with_content_length(val: &str) -> HeaderMap { let mut map = HeaderMap::new(); map.insert(header::CONTENT_LENGTH, val.parse().unwrap()); map } - async fn get_response_body_as_string(response: hyper_migration::HttpResponse) -> String { + async fn get_response_body_as_string(response: http_common::HttpResponse) -> String { let body = response.into_body(); let bytes = body.collect().await.unwrap().to_bytes(); String::from_utf8(bytes.into_iter().collect()).unwrap() diff --git a/crates/datadog-trace-agent/src/mini_agent.rs b/crates/datadog-trace-agent/src/mini_agent.rs index 855290c..ae07481 100644 --- a/crates/datadog-trace-agent/src/mini_agent.rs +++ b/crates/datadog-trace-agent/src/mini_agent.rs @@ -4,7 +4,7 @@ use http_body_util::BodyExt; use hyper::service::service_fn; use hyper::{Method, Response, StatusCode, http}; -use libdd_common::hyper_migration; +use libdd_common::http_common; use serde_json::json; use std::io; use std::net::SocketAddr; @@ -31,6 +31,13 @@ const PROFILING_ENDPOINT_PATH: &str = "/profiling/v1/input"; const TRACER_PAYLOAD_CHANNEL_BUFFER_SIZE: usize = 10; const STATS_PAYLOAD_CHANNEL_BUFFER_SIZE: usize = 10; const PROXY_PAYLOAD_CHANNEL_BUFFER_SIZE: usize = 10; +/// Sentinel file written on startup in Lambda Lite mode. +/// dd-trace (Node.js) checks this path via DATADOG_MINI_AGENT_PATH in constants.js +/// (datadog/dd-trace-js) to decide whether to switch from LogExporter (stdout) to +/// AgentExporter (HTTP :8126). +/// The parent directory `/tmp/datadog/` is created by the serverless-compat JS layer +/// before this binary is spawned. +const LAMBDA_LITE_SENTINEL_PATH: &str = "/tmp/datadog/mini_agent_ready"; pub struct MiniAgent { pub config: Arc, @@ -118,7 +125,7 @@ impl MiniAgent { MiniAgent::trace_endpoint_handler( endpoint_config, - req.map(hyper_migration::Body::incoming), + req.map(http_common::Body::incoming), trace_processor, trace_tx, stats_processor, @@ -173,6 +180,36 @@ impl MiniAgent { let addr = SocketAddr::from(([127, 0, 0, 1], self.config.dd_apm_receiver_port)); let listener = tokio::net::TcpListener::bind(&addr).await?; + // Write the sentinel file after the listener is bound so that dd-trace + // (Node.js) only switches from LogExporter (stdout) to AgentExporter + // (HTTP :8126) once the port is actually ready to accept connections. + // Only written for Lambda Lite; standard Lambda invocations use the + // Extension path (/opt/extensions/datadog-agent) instead. + // /opt is read-only in Lambda Lite, so we use /tmp/datadog/ (created + // by the serverless-compat JS layer before spawning this binary). + if crate::http_utils::is_lambda_lite() { + let sentinel = std::path::Path::new(LAMBDA_LITE_SENTINEL_PATH); + // SAFETY: LAMBDA_LITE_SENTINEL_PATH is a hard-coded absolute path, + // so .parent() always returns Some. + if let Some(parent) = sentinel.parent() + && let Err(e) = tokio::fs::create_dir_all(parent).await + { + error!( + "Could not create parent directory for Lambda Lite sentinel \ + file at {}: {}.", + LAMBDA_LITE_SENTINEL_PATH, e + ); + } + if let Err(e) = tokio::fs::write(sentinel, b"").await { + error!( + "Could not write Lambda Lite sentinel file at {}: {}. \ + dd-trace (Node.js) will fall back to LogExporter (stdout), \ + traces may not reach Datadog.", + LAMBDA_LITE_SENTINEL_PATH, e + ); + } + } + Self::serve_tcp( listener, service, @@ -194,7 +231,7 @@ impl MiniAgent { where S: hyper::service::Service< hyper::Request, - Response = hyper::Response, + Response = hyper::Response, > + Clone + Send + 'static, @@ -265,7 +302,7 @@ impl MiniAgent { where S: hyper::service::Service< hyper::Request, - Response = hyper::Response, + Response = hyper::Response, > + Clone + Send + 'static, @@ -348,14 +385,14 @@ impl MiniAgent { #[allow(clippy::too_many_arguments)] async fn trace_endpoint_handler( config: Arc, - req: hyper_migration::HttpRequest, + req: http_common::HttpRequest, trace_processor: Arc, trace_tx: Sender, stats_processor: Arc, stats_tx: Sender, mini_agent_metadata: Arc, proxy_tx: Sender, - ) -> http::Result { + ) -> http::Result { match (req.method(), req.uri().path()) { (&Method::PUT | &Method::POST, TRACE_ENDPOINT_PATH) => { match trace_processor @@ -412,9 +449,9 @@ impl MiniAgent { /// Handles incoming proxy requests for profiling - can be abstracted into a generic proxy handler for other proxy requests in the future async fn profiling_proxy_handler( config: Arc, - request: hyper_migration::HttpRequest, + request: http_common::HttpRequest, proxy_tx: Sender, - ) -> http::Result { + ) -> http::Result { debug!("Received profiling request"); // Extract headers and body @@ -466,7 +503,7 @@ impl MiniAgent { dd_apm_receiver_port: u16, dd_apm_windows_pipe_name: Option<&str>, dd_dogstatsd_port: u16, - ) -> http::Result { + ) -> http::Result { // pipe_name already includes \\.\pipe\ prefix from config let receiver_socket = dd_apm_windows_pipe_name.unwrap_or(""); @@ -490,6 +527,6 @@ impl MiniAgent { ); Response::builder() .status(200) - .body(hyper_migration::Body::from(response_json.to_string())) + .body(http_common::Body::from(response_json.to_string())) } } diff --git a/crates/datadog-trace-agent/src/stats_flusher.rs b/crates/datadog-trace-agent/src/stats_flusher.rs index 6c6e580..3bee237 100644 --- a/crates/datadog-trace-agent/src/stats_flusher.rs +++ b/crates/datadog-trace-agent/src/stats_flusher.rs @@ -2,6 +2,7 @@ // SPDX-License-Identifier: Apache-2.0 use async_trait::async_trait; +use libdd_common::DefaultHttpClient; use std::{sync::Arc, time}; use tokio::sync::{Mutex, mpsc::Receiver}; use tracing::{debug, error}; @@ -76,7 +77,7 @@ impl StatsFlusher for ServerlessStatsFlusher { }; #[allow(clippy::unwrap_used)] - match stats_utils::send_stats_payload( + match stats_utils::send_stats_payload::( serialized_stats_payload, &config.trace_stats_intake, config.trace_stats_intake.api_key.as_ref().unwrap(), diff --git a/crates/datadog-trace-agent/src/stats_processor.rs b/crates/datadog-trace-agent/src/stats_processor.rs index 889e5f2..aa47a41 100644 --- a/crates/datadog-trace-agent/src/stats_processor.rs +++ b/crates/datadog-trace-agent/src/stats_processor.rs @@ -6,7 +6,7 @@ use std::time::UNIX_EPOCH; use async_trait::async_trait; use hyper::{StatusCode, http}; -use libdd_common::hyper_migration; +use libdd_common::http_common; use tokio::sync::mpsc::Sender; use tracing::debug; @@ -23,9 +23,9 @@ pub trait StatsProcessor { async fn process_stats( &self, config: Arc, - req: hyper_migration::HttpRequest, + req: http_common::HttpRequest, tx: Sender, - ) -> http::Result; + ) -> http::Result; } #[derive(Clone)] @@ -36,9 +36,9 @@ impl StatsProcessor for ServerlessStatsProcessor { async fn process_stats( &self, config: Arc, - req: hyper_migration::HttpRequest, + req: http_common::HttpRequest, tx: Sender, - ) -> http::Result { + ) -> http::Result { debug!("Received trace stats to process"); let (parts, body) = req.into_parts(); diff --git a/crates/datadog-trace-agent/src/trace_flusher.rs b/crates/datadog-trace-agent/src/trace_flusher.rs index cf2619e..638e3fd 100644 --- a/crates/datadog-trace-agent/src/trace_flusher.rs +++ b/crates/datadog-trace-agent/src/trace_flusher.rs @@ -6,7 +6,11 @@ use std::{error::Error, sync::Arc, time}; use tokio::sync::{Mutex, mpsc::Receiver}; use tracing::{debug, error}; -use libdd_common::{GenericHttpClient, hyper_migration}; +use http_body_util::BodyExt; +use libdd_capabilities::http::{HttpClientTrait, HttpError}; +use libdd_capabilities::{MaybeSend, Request, Response}; +use libdd_common::connector::Connector; +use libdd_common::http_common::{self, Body, GenericHttpClient}; use libdd_trace_utils::trace_utils; use libdd_trace_utils::trace_utils::SendData; @@ -75,14 +79,13 @@ impl TraceFlusher for ServerlessTraceFlusher { } debug!("Flushing {} traces", traces.len()); - let http_client = - match ServerlessTraceFlusher::get_http_client(self.config.proxy_url.as_ref()) { - Ok(client) => client, - Err(e) => { - error!("Failed to create HTTP client: {e:?}"); - return; - } - }; + let http_client = match ProxyHttpClient::with_proxy(self.config.proxy_url.as_ref()) { + Ok(client) => client, + Err(e) => { + error!("Failed to create HTTP client: {e:?}"); + return; + } + }; // Retries are handled internally by SendData::send() for coalesced_traces in trace_utils::coalesce_send_data(traces) { @@ -97,26 +100,63 @@ impl TraceFlusher for ServerlessTraceFlusher { } } -impl ServerlessTraceFlusher { - fn get_http_client( - proxy_https: Option<&String>, - ) -> Result< - GenericHttpClient>, - Box, - > { +#[derive(Clone)] +struct ProxyHttpClient { + client: GenericHttpClient>, +} + +impl std::fmt::Debug for ProxyHttpClient { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("ProxyHttpClient").finish() + } +} + +impl ProxyHttpClient { + // `HttpClientTrait::new_client` takes no arguments, so we use `with_proxy` to + // take in the proxy URL and build the client. `new_client` is never called on our code path. + fn with_proxy(proxy_https: Option<&String>) -> Result> { if let Some(proxy) = proxy_https { let proxy = hyper_http_proxy::Proxy::new(hyper_http_proxy::Intercept::Https, proxy.parse()?); - let proxy_connector = hyper_http_proxy::ProxyConnector::from_proxy( - libdd_common::connector::Connector::default(), - proxy, - )?; - Ok(hyper_migration::client_builder().build(proxy_connector)) + let proxy_connector = + hyper_http_proxy::ProxyConnector::from_proxy(Connector::default(), proxy)?; + Ok(Self { + client: http_common::client_builder().build(proxy_connector), + }) } else { - let proxy_connector = hyper_http_proxy::ProxyConnector::new( - libdd_common::connector::Connector::default(), - )?; - Ok(hyper_migration::client_builder().build(proxy_connector)) + let proxy_connector = hyper_http_proxy::ProxyConnector::new(Connector::default())?; + Ok(Self { + client: http_common::client_builder().build(proxy_connector), + }) + } + } +} + +impl HttpClientTrait for ProxyHttpClient { + #[allow(clippy::expect_used)] + fn new_client() -> Self { + Self::with_proxy(None).expect("building proxy connector with default TLS should not fail") + } + + fn request( + &self, + req: Request, + ) -> impl std::future::Future, HttpError>> + MaybeSend + { + let client = self.client.clone(); + async move { + let hyper_req = req.map(Body::from_bytes); + let response = client + .request(hyper_req) + .await + .map_err(|e| HttpError::Network(e.into()))?; + let (parts, body) = response.into_parts(); + let collected = body + .collect() + .await + .map_err(|e| HttpError::ResponseBody(e.into()))? + .to_bytes(); + Ok(Response::from_parts(parts, collected)) } } } diff --git a/crates/datadog-trace-agent/src/trace_processor.rs b/crates/datadog-trace-agent/src/trace_processor.rs index 1685137..96f8209 100644 --- a/crates/datadog-trace-agent/src/trace_processor.rs +++ b/crates/datadog-trace-agent/src/trace_processor.rs @@ -5,7 +5,7 @@ use std::sync::Arc; use async_trait::async_trait; use hyper::{StatusCode, http}; -use libdd_common::hyper_migration; +use libdd_common::http_common; use tokio::sync::mpsc::Sender; use tracing::debug; @@ -29,10 +29,10 @@ pub trait TraceProcessor { async fn process_traces( &self, config: Arc, - req: hyper_migration::HttpRequest, + req: http_common::HttpRequest, tx: Sender, mini_agent_metadata: Arc, - ) -> http::Result; + ) -> http::Result; } struct ChunkProcessor { @@ -72,10 +72,10 @@ impl TraceProcessor for ServerlessTraceProcessor { async fn process_traces( &self, config: Arc, - req: hyper_migration::HttpRequest, + req: http_common::HttpRequest, tx: Sender, mini_agent_metadata: Arc, - ) -> http::Result { + ) -> http::Result { debug!("Received traces to process"); let (parts, body) = req.into_parts(); @@ -170,7 +170,7 @@ mod tests { config::{Config, Tags}, trace_processor::{self, TRACER_PAYLOAD_FUNCTION_TAGS_TAG_KEY, TraceProcessor}, }; - use libdd_common::{Endpoint, hyper_migration}; + use libdd_common::{Endpoint, http_common}; use libdd_trace_protobuf::pb; use libdd_trace_utils::test_utils::{create_test_gcp_json_span, create_test_gcp_span}; use libdd_trace_utils::trace_utils::MiniAgentMetadata; @@ -251,7 +251,7 @@ mod tests { .header("datadog-meta-lang-interpreter", "v8") .header("datadog-container-id", "33") .header("content-length", "100") - .body(hyper_migration::Body::from(bytes)) + .body(http_common::Body::from(bytes)) .unwrap(); let trace_processor = trace_processor::ServerlessTraceProcessor {}; @@ -323,7 +323,7 @@ mod tests { .header("datadog-meta-lang-interpreter", "v8") .header("datadog-container-id", "33") .header("content-length", "100") - .body(hyper_migration::Body::from(bytes)) + .body(http_common::Body::from(bytes)) .unwrap(); let trace_processor = trace_processor::ServerlessTraceProcessor {}; diff --git a/crates/datadog-trace-agent/tests/common/helpers.rs b/crates/datadog-trace-agent/tests/common/helpers.rs index 6f6e776..6dd8d82 100644 --- a/crates/datadog-trace-agent/tests/common/helpers.rs +++ b/crates/datadog-trace-agent/tests/common/helpers.rs @@ -5,7 +5,7 @@ use hyper::{Request, Response}; use hyper_util::rt::TokioIo; -use libdd_common::hyper_migration; +use libdd_common::http_common; use libdd_trace_utils::test_utils::create_test_json_span; use std::time::{Duration, UNIX_EPOCH}; use tokio::time::timeout; @@ -45,10 +45,10 @@ pub async fn send_tcp_request( let response = if let Some(body_data) = body { let body_len = body_data.len(); request_builder = request_builder.header("Content-Length", body_len.to_string()); - let request = request_builder.body(hyper_migration::Body::from(body_data))?; + let request = request_builder.body(http_common::Body::from(body_data))?; timeout(Duration::from_secs(2), sender.send_request(request)).await?? } else { - let request = request_builder.body(hyper_migration::Body::empty())?; + let request = request_builder.body(http_common::Body::empty())?; timeout(Duration::from_secs(2), sender.send_request(request)).await?? }; @@ -83,10 +83,10 @@ pub async fn send_named_pipe_request( let response = if let Some(body_data) = body { let body_len = body_data.len(); request_builder = request_builder.header("Content-Length", body_len.to_string()); - let request = request_builder.body(hyper_migration::Body::from(body_data))?; + let request = request_builder.body(http_common::Body::from(body_data))?; timeout(Duration::from_secs(2), sender.send_request(request)).await?? } else { - let request = request_builder.body(hyper_migration::Body::empty())?; + let request = request_builder.body(http_common::Body::empty())?; timeout(Duration::from_secs(2), sender.send_request(request)).await?? }; diff --git a/crates/datadog-trace-agent/tests/common/mock_server.rs b/crates/datadog-trace-agent/tests/common/mock_server.rs index b78b96c..f1beb1a 100644 --- a/crates/datadog-trace-agent/tests/common/mock_server.rs +++ b/crates/datadog-trace-agent/tests/common/mock_server.rs @@ -6,7 +6,7 @@ use http_body_util::BodyExt; use hyper::{Request, Response, body::Incoming}; use hyper_util::rt::TokioIo; -use libdd_common::hyper_migration; +use libdd_common::http_common; use std::net::SocketAddr; use std::sync::{Arc, Mutex}; use tokio::net::TcpListener; @@ -86,7 +86,7 @@ impl MockServer { Ok::<_, hyper::http::Error>( Response::builder() .status(200) - .body(hyper_migration::Body::from(r#"{"ok":true}"#)) + .body(http_common::Body::from(r#"{"ok":true}"#)) .unwrap(), ) } diff --git a/crates/datadog-trace-agent/tests/common/mocks.rs b/crates/datadog-trace-agent/tests/common/mocks.rs index d57aa2a..842c45f 100644 --- a/crates/datadog-trace-agent/tests/common/mocks.rs +++ b/crates/datadog-trace-agent/tests/common/mocks.rs @@ -7,7 +7,7 @@ use datadog_trace_agent::{ config::Config, env_verifier::EnvVerifier, stats_flusher::StatsFlusher, stats_processor::StatsProcessor, trace_flusher::TraceFlusher, trace_processor::TraceProcessor, }; -use libdd_common::hyper_migration; +use libdd_common::http_common; use libdd_trace_protobuf::pb; use libdd_trace_utils::trace_utils::{self, MiniAgentMetadata, SendData}; use std::sync::Arc; @@ -22,13 +22,13 @@ impl TraceProcessor for MockTraceProcessor { async fn process_traces( &self, _config: Arc, - _req: hyper_migration::HttpRequest, + _req: http_common::HttpRequest, _trace_tx: Sender, _mini_agent_metadata: Arc, - ) -> Result { + ) -> Result { hyper::Response::builder() .status(200) - .body(hyper_migration::Body::from("{}")) + .body(http_common::Body::from("{}")) } } @@ -68,12 +68,12 @@ impl StatsProcessor for MockStatsProcessor { async fn process_stats( &self, _config: Arc, - _req: hyper_migration::HttpRequest, + _req: http_common::HttpRequest, _stats_tx: Sender, - ) -> Result { + ) -> Result { hyper::Response::builder() .status(200) - .body(hyper_migration::Body::from("{}")) + .body(http_common::Body::from("{}")) } } diff --git a/crates/datadog-trace-agent/tests/integration_test.rs b/crates/datadog-trace-agent/tests/integration_test.rs index bf28d4f..1491954 100644 --- a/crates/datadog-trace-agent/tests/integration_test.rs +++ b/crates/datadog-trace-agent/tests/integration_test.rs @@ -295,11 +295,11 @@ async fn test_mini_agent_tcp_with_real_flushers() { let mut server_ready = false; for _ in 0..20 { tokio::time::sleep(Duration::from_millis(50)).await; - if let Ok(response) = send_tcp_request(test_port, "/info", "GET", None).await { - if response.status().is_success() { - server_ready = true; - break; - } + if let Ok(response) = send_tcp_request(test_port, "/info", "GET", None).await + && response.status().is_success() + { + server_ready = true; + break; } } assert!( diff --git a/crates/dogstatsd/src/flusher.rs b/crates/dogstatsd/src/flusher.rs index d26579f..70032c6 100644 --- a/crates/dogstatsd/src/flusher.rs +++ b/crates/dogstatsd/src/flusher.rs @@ -231,6 +231,7 @@ async fn should_try_next_batch(resp: Result) -> (bool, } #[cfg(test)] +#[allow(clippy::disallowed_methods)] mod tests { use super::*; use crate::aggregator::AggregatorService; diff --git a/crates/dogstatsd/src/origin.rs b/crates/dogstatsd/src/origin.rs index d0c0952..fc025b9 100644 --- a/crates/dogstatsd/src/origin.rs +++ b/crates/dogstatsd/src/origin.rs @@ -162,16 +162,10 @@ mod tests { timestamp: 0, }; let origin = metric.find_origin(tags).unwrap(); + assert_eq!(origin.origin_product, OriginProduct::Serverless as u32); + assert_eq!(origin.origin_category, OriginCategory::LambdaMetrics as u32); assert_eq!( - origin.origin_product as u32, - OriginProduct::Serverless as u32 - ); - assert_eq!( - origin.origin_category as u32, - OriginCategory::LambdaMetrics as u32 - ); - assert_eq!( - origin.origin_service as u32, + origin.origin_service, OriginService::ServerlessRuntime as u32 ); } @@ -187,16 +181,10 @@ mod tests { timestamp: 0, }; let origin = metric.find_origin(tags).unwrap(); + assert_eq!(origin.origin_product, OriginProduct::Serverless as u32); + assert_eq!(origin.origin_category, OriginCategory::LambdaMetrics as u32); assert_eq!( - origin.origin_product as u32, - OriginProduct::Serverless as u32 - ); - assert_eq!( - origin.origin_category as u32, - OriginCategory::LambdaMetrics as u32 - ); - assert_eq!( - origin.origin_service as u32, + origin.origin_service, OriginService::ServerlessEnhanced as u32 ); } @@ -212,16 +200,10 @@ mod tests { timestamp: 0, }; let origin = metric.find_origin(tags).unwrap(); + assert_eq!(origin.origin_product, OriginProduct::Serverless as u32); + assert_eq!(origin.origin_category, OriginCategory::LambdaMetrics as u32); assert_eq!( - origin.origin_product as u32, - OriginProduct::Serverless as u32 - ); - assert_eq!( - origin.origin_category as u32, - OriginCategory::LambdaMetrics as u32 - ); - assert_eq!( - origin.origin_service as u32, + origin.origin_service, OriginService::ServerlessCustom as u32 ); } @@ -237,16 +219,13 @@ mod tests { timestamp: 0, }; let origin = metric.find_origin(tags).unwrap(); + assert_eq!(origin.origin_product, OriginProduct::Serverless as u32); assert_eq!( - origin.origin_product as u32, - OriginProduct::Serverless as u32 - ); - assert_eq!( - origin.origin_category as u32, + origin.origin_category, OriginCategory::CloudRunMetrics as u32 ); assert_eq!( - origin.origin_service as u32, + origin.origin_service, OriginService::ServerlessEnhanced as u32 ); } @@ -262,16 +241,13 @@ mod tests { timestamp: 0, }; let origin = metric.find_origin(tags).unwrap(); + assert_eq!(origin.origin_product, OriginProduct::Serverless as u32); assert_eq!( - origin.origin_product as u32, - OriginProduct::Serverless as u32 - ); - assert_eq!( - origin.origin_category as u32, + origin.origin_category, OriginCategory::CloudRunMetrics as u32 ); assert_eq!( - origin.origin_service as u32, + origin.origin_service, OriginService::ServerlessCustom as u32 ); } @@ -287,16 +263,13 @@ mod tests { timestamp: 0, }; let origin = metric.find_origin(tags).unwrap(); + assert_eq!(origin.origin_product, OriginProduct::Serverless as u32); assert_eq!( - origin.origin_product as u32, - OriginProduct::Serverless as u32 - ); - assert_eq!( - origin.origin_category as u32, + origin.origin_category, OriginCategory::AppServicesMetrics as u32 ); assert_eq!( - origin.origin_service as u32, + origin.origin_service, OriginService::ServerlessCustom as u32 ); } @@ -312,16 +285,13 @@ mod tests { timestamp: 0, }; let origin = metric.find_origin(tags).unwrap(); + assert_eq!(origin.origin_product, OriginProduct::Serverless as u32); assert_eq!( - origin.origin_product as u32, - OriginProduct::Serverless as u32 - ); - assert_eq!( - origin.origin_category as u32, + origin.origin_category, OriginCategory::ContainerAppMetrics as u32 ); assert_eq!( - origin.origin_service as u32, + origin.origin_service, OriginService::ServerlessCustom as u32 ); } @@ -337,16 +307,13 @@ mod tests { timestamp: 0, }; let origin = metric.find_origin(tags).unwrap(); + assert_eq!(origin.origin_product, OriginProduct::Serverless as u32); assert_eq!( - origin.origin_product as u32, - OriginProduct::Serverless as u32 - ); - assert_eq!( - origin.origin_category as u32, + origin.origin_category, OriginCategory::AzureFunctionsMetrics as u32 ); assert_eq!( - origin.origin_service as u32, + origin.origin_service, OriginService::ServerlessCustom as u32 ); } @@ -362,16 +329,10 @@ mod tests { timestamp: 0, }; let origin = metric.find_origin(tags).unwrap(); + assert_eq!(origin.origin_product, OriginProduct::Serverless as u32); + assert_eq!(origin.origin_category, OriginCategory::LambdaMetrics as u32); assert_eq!( - origin.origin_product as u32, - OriginProduct::Serverless as u32 - ); - assert_eq!( - origin.origin_category as u32, - OriginCategory::LambdaMetrics as u32 - ); - assert_eq!( - origin.origin_service as u32, + origin.origin_service, OriginService::ServerlessRuntime as u32 ); } diff --git a/crates/dogstatsd/tests/integration_test.rs b/crates/dogstatsd/tests/integration_test.rs index 3155ef8..f390fa4 100644 --- a/crates/dogstatsd/tests/integration_test.rs +++ b/crates/dogstatsd/tests/integration_test.rs @@ -1,5 +1,6 @@ // Copyright 2023-Present Datadog, Inc. https://www.datadoghq.com/ // SPDX-License-Identifier: Apache-2.0 +#![allow(clippy::disallowed_methods)] use dogstatsd::metric::SortedTags; use dogstatsd::{ diff --git a/npm/datadog-serverless-compat-darwin-arm64/package.json b/npm/datadog-serverless-compat-darwin-arm64/package.json new file mode 100644 index 0000000..b4211e1 --- /dev/null +++ b/npm/datadog-serverless-compat-darwin-arm64/package.json @@ -0,0 +1,25 @@ +{ + "name": "@datadog/serverless-compat-darwin-arm64", + "version": "0.0.0", + "description": "macOS arm64 binary for the Datadog Serverless Compatibility Layer", + "os": [ + "darwin" + ], + "cpu": [ + "arm64" + ], + "files": [ + "bin/" + ], + "publishConfig": { + "access": "public", + "executableFiles": [ + "./bin/datadog-serverless-compat" + ] + }, + "license": "Apache-2.0", + "repository": { + "type": "git", + "url": "https://github.com/DataDog/serverless-components" + } +} diff --git a/npm/datadog-serverless-compat-win32-ia32/package.json b/npm/datadog-serverless-compat-win32-ia32/package.json new file mode 100644 index 0000000..7744c7b --- /dev/null +++ b/npm/datadog-serverless-compat-win32-ia32/package.json @@ -0,0 +1,22 @@ +{ + "name": "@datadog/serverless-compat-win32-ia32", + "version": "0.0.0", + "description": "Windows ia32 binary for the Datadog Serverless Compatibility Layer", + "os": [ + "win32" + ], + "cpu": [ + "ia32" + ], + "files": [ + "bin/" + ], + "publishConfig": { + "access": "public" + }, + "license": "Apache-2.0", + "repository": { + "type": "git", + "url": "https://github.com/DataDog/serverless-components" + } +} diff --git a/scripts/test-log-intake.sh b/scripts/test-log-intake.sh new file mode 100755 index 0000000..1579aa2 --- /dev/null +++ b/scripts/test-log-intake.sh @@ -0,0 +1,134 @@ +#!/usr/bin/env bash +set -x +# Copyright 2023-Present Datadog, Inc. https://www.datadoghq.com/ +# SPDX-License-Identifier: Apache-2.0 +# +# test-log-intake.sh — Run the log agent example against a local capture server. +# +# The script starts a tiny Python HTTP server that prints every incoming request +# body to stdout so you can inspect the JSON payloads the log agent sends. +# +# USAGE +# # Local capture (default) — no real Datadog traffic: +# ./scripts/test-log-intake.sh +# +# # Send N entries instead of the default 5: +# LOG_ENTRY_COUNT=50 ./scripts/test-log-intake.sh +# +# # Flush to a real Datadog endpoint instead of the local server: +# DD_API_KEY= ./scripts/test-log-intake.sh --real +# +# REQUIREMENTS +# python3 (macOS system python is fine) +# cargo + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +WORKSPACE_ROOT="$(cd "${SCRIPT_DIR}/.." && pwd)" +PORT="${LOG_CAPTURE_PORT:-9999}" +REAL_MODE=false + +for arg in "$@"; do + [[ "$arg" == "--real" ]] && REAL_MODE=true +done + +# ── Build the example first ────────────────────────────────────────────────── +echo "Building send_logs example..." +cargo build -p datadog-logs-agent --example send_logs --quiet 2>&1 + +# ── Real Datadog mode ───────────────────────────────────────────────────────── +if [[ "$REAL_MODE" == true ]]; then + if [[ -z "${DD_API_KEY:-}" ]]; then + echo "Error: DD_API_KEY must be set for --real mode" >&2 + exit 1 + fi + echo "" + echo "Flushing to real Datadog endpoint..." + LOG_ENTRY_COUNT="${LOG_ENTRY_COUNT:-5}" \ + cargo run -p datadog-logs-agent --example send_logs --quiet 2>&1 + exit $? +fi + +# ── Local capture server mode ───────────────────────────────────────────────── + +# Python HTTP server that prints the request body as formatted JSON +CAPTURE_SERVER_SCRIPT=$(cat <<'PYEOF' +import http.server +import json +import sys + +class Handler(http.server.BaseHTTPRequestHandler): + def do_POST(self): + length = int(self.headers.get("Content-Length", 0)) + body = self.rfile.read(length) + encoding = self.headers.get("Content-Encoding", "none") + content_type = self.headers.get("Content-Type", "") + + print(f"\n{'─'*60}") + print(f"POST {self.path}") + print(f"DD-API-KEY : {self.headers.get('DD-API-KEY', '(not set)')}") + print(f"DD-PROTOCOL: {self.headers.get('DD-PROTOCOL', '(not set)')}") + print(f"Content-Encoding: {encoding}") + print(f"Content-Type : {content_type}") + + if encoding == "zstd": + try: + import zstd + body = zstd.decompress(body) + print("(decompressed zstd payload)") + except ImportError: + print("(zstd payload — install python-zstd to decompress: pip install zstd)") + + if "json" in content_type or body.startswith(b"[") or body.startswith(b"{"): + try: + parsed = json.loads(body) + print(f"\nPayload ({len(parsed) if isinstance(parsed, list) else 1} entries):") + print(json.dumps(parsed, indent=2)) + except json.JSONDecodeError: + print(f"\nRaw body ({len(body)} bytes): {body[:500]}") + else: + print(f"\nRaw body ({len(body)} bytes)") + + self.send_response(200) + self.end_headers() + sys.stdout.flush() + + def log_message(self, fmt, *args): + pass # suppress default access log noise + +port = int(sys.argv[1]) +print(f"Capture server listening on http://localhost:{port}") +print("Waiting for log flush... (Ctrl-C to stop)\n") +sys.stdout.flush() + +httpd = http.server.HTTPServer(("localhost", port), Handler) +httpd.serve_forever() +PYEOF +) + +# Start capture server in background +python3 -c "$CAPTURE_SERVER_SCRIPT" "$PORT" & +SERVER_PID=$! + +cleanup() { + kill "$SERVER_PID" 2>/dev/null || true +} +trap cleanup EXIT INT TERM + +# Give the server a moment to start +sleep 0.3 + +echo "" +echo "Running send_logs example → http://localhost:${PORT}/logs" +echo "" + +DD_OBSERVABILITY_PIPELINES_WORKER_LOGS_ENABLED=true \ +DD_OBSERVABILITY_PIPELINES_WORKER_LOGS_URL="http://localhost:${PORT}/logs" \ +DD_API_KEY="${DD_API_KEY:-local-test-key}" \ +LOG_ENTRY_COUNT="${LOG_ENTRY_COUNT:-5}" \ + cargo run -p datadog-logs-agent --example send_logs --quiet 2>&1 + +echo "" +echo "Done. Press Ctrl-C to stop the capture server." +wait "$SERVER_PID"