diff --git a/.codecov.yml b/.codecov.yml index d11d93b364..452c5b3c0b 100644 --- a/.codecov.yml +++ b/.codecov.yml @@ -23,6 +23,10 @@ coverage: target: 75 flags: - bind9 + Claude_Enterprise_Analytics: + target: 75 + flags: + - claude_enterprise_analytics CloudNatix: target: 75 flags: @@ -374,6 +378,11 @@ flags: paths: - cfssl/datadog_checks/cfssl - cfssl/tests + claude_enterprise_analytics: + carryforward: true + paths: + - claude_enterprise_analytics/datadog_checks/claude_enterprise_analytics + - claude_enterprise_analytics/tests cloudnatix: carryforward: true paths: diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 3412e2a8b5..033e906b00 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -1183,6 +1183,8 @@ code-coverage.datadog.yml @DataDog/agent-integr /dagster/ @DataDog/ecosystems-review /aerospike_enterprise/ @support-aerospike @DataDog/ecosystems-review /scamalytics/ @ScamalyticsDev @DataDog/ecosystems-review +/claude_enterprise_analytics/ @matiasozdy @DataDog/ecosystems-review +/claude_enterprise_analytics/*metadata.csv @matiasozdy @DataDog/documentation @DataDog/ecosystems-review # LEAVE THE FOLLOWING LOG OWNERSHIP LAST IN THE FILE # Make sure logs team is the full owner for all logs related files diff --git a/.github/workflows/test-all.yml b/.github/workflows/test-all.yml index 7a5be903a9..7ff33cb15e 100644 --- a/.github/workflows/test-all.yml +++ b/.github/workflows/test-all.yml @@ -158,6 +158,25 @@ jobs: test-py3: ${{ inputs.test-py3 }} setup-env-vars: "${{ inputs.setup-env-vars }}" secrets: inherit + jf0663d2: + uses: DataDog/integrations-core/.github/workflows/test-target.yml@574d63ba88365ffbab915280ceddbaa333c63d6a + with: + job-name: Claude Enterprise Analytics + target: claude_enterprise_analytics + platform: linux + runner: '["ubuntu-22.04"]' + repo: "${{ inputs.repo }}" + context: ${{ inputs.context }} + python-version: "${{ inputs.python-version }}" + latest: ${{ inputs.latest }} + agent-image: "${{ inputs.agent-image }}" + agent-image-py2: "${{ inputs.agent-image-py2 }}" + agent-image-windows: "${{ inputs.agent-image-windows }}" + agent-image-windows-py2: "${{ inputs.agent-image-windows-py2 }}" + test-py2: ${{ inputs.test-py2 }} + test-py3: ${{ inputs.test-py3 }} + setup-env-vars: "${{ inputs.setup-env-vars }}" + secrets: inherit j6a8ad70: uses: DataDog/integrations-core/.github/workflows/test-target.yml@574d63ba88365ffbab915280ceddbaa333c63d6a with: diff --git a/claude_enterprise_analytics/CHANGELOG.md b/claude_enterprise_analytics/CHANGELOG.md new file mode 100644 index 0000000000..7679264909 --- /dev/null +++ b/claude_enterprise_analytics/CHANGELOG.md @@ -0,0 +1,4 @@ +# CHANGELOG - Claude Enterprise Analytics + + + diff --git a/claude_enterprise_analytics/README.md b/claude_enterprise_analytics/README.md new file mode 100644 index 0000000000..f02d7514be --- /dev/null +++ b/claude_enterprise_analytics/README.md @@ -0,0 +1,94 @@ +# Agent Check: Claude Enterprise Analytics + +> **Unofficial integration.** This check is community-maintained and **not affiliated with, endorsed by, or supported by Anthropic, PBC**. "Anthropic" and "Claude" are trademarks of Anthropic, PBC. For official support of the underlying API, contact Anthropic. + +## Overview + +This integration pulls daily usage, cost, and seat-utilization data from the [Anthropic Claude Enterprise Analytics API][1] and submits it to Datadog as metrics. It is useful for Anthropic enterprise customers who want Claude usage observability inside their existing Datadog dashboards: cost attribution per user/model/product, seat adoption tracking, Claude Code activity (commits, PRs, lines, tool acceptance rate), token mix and cache effectiveness, and web-search usage. + +The check polls six API endpoints once per collection interval for a single past day (default: today minus 3, because Anthropic's analytics have a 3-day aggregation lag): + +- `/summaries` -- org-wide DAU/WAU/MAU, assigned seats, adoption rates +- `/users` -- per-user activity across chat, projects, artifacts, Claude Code, web search +- `/usage_report` -- token consumption by model and product +- `/cost_report` -- USD spend by model and product (list price; see "Known Limitations") +- `/user_usage_report` -- per-user token totals +- `/user_cost_report` -- per-user USD spend + +All metrics are emitted as gauges at agent wall-clock time and tagged with `report_date:YYYY-MM-DD` representing the activity day. Build dashboards that filter or group by `report_date` rather than relying on Datadog's wall-clock x-axis. + +## Setup + +### Installation + +1. Install the [Datadog Agent][2] on your host. +2. Install this integration from the `integrations-extras` repo: + ``` + datadog-agent integration install -t datadog-claude-enterprise-analytics== + ``` + +### Configuration + +1. Generate an API key at [claude.ai/analytics/api-keys][1] (requires the Primary Owner role on your Anthropic org). The key must carry the `read:analytics` scope. +2. Edit `claude_enterprise_analytics.d/conf.yaml` in your Agent's `conf.d/` directory: + ```yaml + instances: + - anthropic_api_key: + org_id: my-org # free-form tag value, surfaced on every metric + lag_days: 3 # how many days behind today to poll + min_collection_interval: 3600 # once per hour is plenty; Anthropic refreshes daily + ``` + See [`conf.yaml.example`][4] for all options. +3. [Restart the Agent][5]. + +### Validation + +Run [`datadog-agent status`][6] and look for `claude_enterprise_analytics` under the Checks section. Then in the Datadog UI, open Metrics Explorer and search for `claude_enterprise_analytics.*` -- you should see ~45 metrics arrive within one collection interval. + +## Data Collected + +### Metrics + +45 gauges under the `claude_enterprise_analytics.` namespace. Highlights: + +- `claude_enterprise_analytics.org.dau` / `.wau` / `.mau` / `.seats_assigned` / `.adoption_rate.{daily,weekly,monthly}` +- `claude_enterprise_analytics.cost.amount_usd` / `.list_amount_usd` (tagged by `model`, `product`) +- `claude_enterprise_analytics.user.cost.amount_usd` (tagged by `user_email`) +- `claude_enterprise_analytics.tokens.{uncached_input,output,cache_read,cache_write_1h,cache_write_5m}` +- `claude_enterprise_analytics.user.claude_code.{sessions,commits,prs,lines_added,lines_removed,tool_actions}` +- `claude_enterprise_analytics.web_search_requests` + +See [`metadata.csv`][7] for the full list with descriptions and tag keys. + +### Events + +This integration does not submit events. + +### Service Checks + +- `claude_enterprise_analytics.can_connect` -- `CRITICAL` if the Anthropic API could not be reached during the check run, otherwise `OK`. See [`service_checks.json`][8]. + +## Known Limitations + +- **Cost is list-priced.** Anthropic's API returns `amount` equal to `list_amount` in every response. Enterprise contract discounts are not exposed through this API, so the dashboard's USD figures reflect retail compute consumption rather than your actual invoice. Treat cost data as a relative attribution signal (which model/user is most expensive) rather than as billing truth. +- **3-day data lag.** Each report represents activity from `today - 3` (configurable via `lag_days`). Polling more frequently than once per day will resubmit the same daily totals. +- **Datadog metric submission window.** Metrics are submitted at the agent's current wall-clock time, not the activity date -- Datadog rejects timestamps more than ~1h in the past. The activity date is preserved via the `report_date` tag; dashboards must group/filter by it. +- **Cost is returned in USD cents** (smallest currency subunit) despite the `currency: USD` field. This check divides by 100 before submission. + +## Troubleshooting + +- **No data after a fresh install:** Anthropic's analytics endpoint can return empty bodies for very recent dates. Try increasing `lag_days` from 3 to 5. +- **`401 Unauthorized`:** confirm the API key is generated by a Primary Owner and carries the `read:analytics` scope. +- **`400 Bad Request`** on the first run: the underlying analytics endpoints expect strict-before date ranges; this check handles that automatically, but a mismatched system clock can produce the wrong date. Verify the agent host has accurate UTC time. + +Need help with the integration itself? Open an issue against the [integrations-extras repository][10]. For Anthropic API issues, contact your Anthropic account team. + +[1]: https://support.claude.com/en/articles/13703965-claude-enterprise-analytics-api-reference-guide +[2]: https://app.datadoghq.com/account/settings/agent/latest +[3]: https://docs.datadoghq.com/containers/kubernetes/integrations/ +[4]: https://github.com/DataDog/integrations-extras/blob/master/claude_enterprise_analytics/datadog_checks/claude_enterprise_analytics/data/conf.yaml.example +[5]: https://docs.datadoghq.com/agent/configuration/agent-commands/#start-stop-and-restart-the-agent +[6]: https://docs.datadoghq.com/agent/configuration/agent-commands/#agent-status-and-information +[7]: https://github.com/DataDog/integrations-extras/blob/master/claude_enterprise_analytics/metadata.csv +[8]: https://github.com/DataDog/integrations-extras/blob/master/claude_enterprise_analytics/assets/service_checks.json +[10]: https://github.com/DataDog/integrations-extras/issues diff --git a/claude_enterprise_analytics/assets/configuration/spec.yaml b/claude_enterprise_analytics/assets/configuration/spec.yaml new file mode 100644 index 0000000000..6ea7424e92 --- /dev/null +++ b/claude_enterprise_analytics/assets/configuration/spec.yaml @@ -0,0 +1,38 @@ +name: Claude Enterprise Analytics +files: +- name: claude_enterprise_analytics.yaml + options: + - template: init_config + options: + - template: init_config/default + - template: instances + options: + - name: anthropic_api_key + required: true + secret: true + description: |- + API key for the Anthropic Claude Enterprise Analytics API. Generate one at + https://claude.ai/analytics/api-keys (requires Primary Owner role). Must + carry the `read:analytics` scope. + value: + type: string + example: + - name: org_id + required: false + description: |- + Free-form organization identifier emitted as the `org_id` tag on every + metric. Useful when more than one Anthropic org reports into the same + Datadog account. + value: + type: string + example: my-org + - name: lag_days + required: false + description: |- + How many days behind today to poll. Anthropic publishes daily aggregates + with a roughly 3-day lag; the default reflects that. Increase if you see + empty responses for the default date. + value: + type: integer + example: 3 + - template: instances/default diff --git a/claude_enterprise_analytics/assets/dashboards/claude_enterprise_analytics_overview.json b/claude_enterprise_analytics/assets/dashboards/claude_enterprise_analytics_overview.json new file mode 100644 index 0000000000..a988f5d278 --- /dev/null +++ b/claude_enterprise_analytics/assets/dashboards/claude_enterprise_analytics_overview.json @@ -0,0 +1,2315 @@ +{ + "title": "Claude Enterprise Analytics \u2014 Usage and Costs Overview", + "description": "Monitor Claude usage, token consumption, cache effectiveness, and associated costs.\n\n**Each metric represents one day's totals** from Anthropic's Claude Enterprise Analytics API (3-day data lag). Use the `$report_date` template variable to pin to a specific day; leaving it as `*` shows the sum across **every polled day**, which is typically not what you want.\n\n**Costs are at list price.** `amount` equals `list_amount` in every API response, so the actual enterprise invoice may differ substantially from what's shown here. Cost values were originally returned in cents and converted to USD on the way into Datadog.\n\nMirror of Anthropic's reference 'Usage and Costs' dashboard, wired to the metrics published by typeform/tools/datadog-anthropic-analytics. Unofficial \u2014 not affiliated with Anthropic, PBC.", + "widgets": [ + { + "id": 1001, + "definition": { + "title": "Daily Spend (USD, list price)", + "title_size": "16", + "title_align": "left", + "type": "query_value", + "requests": [ + { + "formulas": [ + { + "formula": "q", + "number_format": { + "unit": { + "type": "canonical_unit", + "unit_name": "dollar" + } + } + } + ], + "queries": [ + { + "data_source": "metrics", + "name": "q", + "query": "sum:claude_enterprise_analytics.cost.amount_usd{$report_date,$org_id,$model,$product,$user_email}", + "aggregator": "last" + } + ], + "response_format": "scalar" + } + ], + "autoscale": true, + "precision": 2 + }, + "layout": { + "x": 0, + "y": 0, + "width": 3, + "height": 2 + } + }, + { + "id": 1002, + "definition": { + "title": "Daily Spend at List Price", + "title_size": "16", + "title_align": "left", + "type": "query_value", + "requests": [ + { + "formulas": [ + { + "formula": "q", + "number_format": { + "unit": { + "type": "canonical_unit", + "unit_name": "dollar" + } + } + } + ], + "queries": [ + { + "data_source": "metrics", + "name": "q", + "query": "sum:claude_enterprise_analytics.cost.list_amount_usd{$report_date,$org_id,$model,$product,$user_email}", + "aggregator": "last" + } + ], + "response_format": "scalar" + } + ], + "autoscale": true, + "precision": 2 + }, + "layout": { + "x": 3, + "y": 0, + "width": 3, + "height": 2 + } + }, + { + "id": 1003, + "definition": { + "title": "Daily Requests", + "title_size": "16", + "title_align": "left", + "type": "query_value", + "requests": [ + { + "formulas": [ + { + "formula": "q" + } + ], + "queries": [ + { + "data_source": "metrics", + "name": "q", + "query": "sum:claude_enterprise_analytics.requests{$report_date,$org_id,$model,$product}", + "aggregator": "last" + } + ], + "response_format": "scalar" + } + ], + "autoscale": true, + "precision": 0 + }, + "layout": { + "x": 6, + "y": 0, + "width": 3, + "height": 2 + } + }, + { + "id": 1004, + "definition": { + "title": "Discount % (API-reported)", + "title_size": "16", + "title_align": "left", + "type": "query_value", + "requests": [ + { + "formulas": [ + { + "formula": "(list - amount) / list * 100", + "number_format": { + "unit": { + "type": "canonical_unit", + "unit_name": "percent" + } + } + } + ], + "queries": [ + { + "data_source": "metrics", + "name": "list", + "query": "sum:claude_enterprise_analytics.cost.list_amount_usd{$report_date,$org_id,$model,$product,$user_email}", + "aggregator": "last" + }, + { + "data_source": "metrics", + "name": "amount", + "query": "sum:claude_enterprise_analytics.cost.amount_usd{$report_date,$org_id,$model,$product,$user_email}", + "aggregator": "last" + } + ], + "response_format": "scalar", + "conditional_formats": [ + { + "comparator": ">", + "value": 0, + "palette": "white_on_green" + } + ] + } + ], + "autoscale": true, + "precision": 1 + }, + "layout": { + "x": 9, + "y": 0, + "width": 3, + "height": 2 + } + }, + { + "id": 4, + "definition": { + "title": "Cost Overview", + "background_color": "vivid_green", + "show_title": true, + "type": "group", + "layout_type": "ordered", + "widgets": [ + { + "id": 407, + "definition": { + "type": "note", + "content": "**Model Cost Efficiency**: Each Claude model has different pricing tiers. Opus models are premium but offer highest quality, Sonnet provides balanced cost-performance, and Haiku is most economical. Analyze cost per task completion to determine optimal model selection for different use cases.\n\nIn Datadog, set up cost alerts on `claude_enterprise_analytics.cost.amount_usd` (e.g. spike or budget threshold per `product`) to catch unexpected expenses. Regular reviews using the `report_date` breakdown help identify trends and anomalies before they impact budgets significantly.", + "background_color": "green", + "font_size": "14", + "text_align": "left", + "show_tick": false + }, + "layout": { + "x": 0, + "y": 0, + "width": 3, + "height": 3 + } + }, + { + "id": 401, + "definition": { + "title": "Daily Cost by Model (by day, USD list price)", + "title_size": "16", + "title_align": "left", + "type": "query_table", + "requests": [ + { + "response_format": "scalar", + "queries": [ + { + "data_source": "metrics", + "name": "amount", + "query": "sum:claude_enterprise_analytics.cost.amount_usd{$report_date,$org_id,$product,$user_email} by {report_date,model}", + "aggregator": "last" + }, + { + "data_source": "metrics", + "name": "list", + "query": "sum:claude_enterprise_analytics.cost.list_amount_usd{$report_date,$org_id,$product,$user_email} by {report_date,model}", + "aggregator": "last" + }, + { + "data_source": "metrics", + "name": "req", + "query": "sum:claude_enterprise_analytics.requests{$report_date,$org_id,$product} by {report_date,model}", + "aggregator": "last" + } + ], + "formulas": [ + { + "alias": "Cost ($)", + "formula": "amount", + "cell_display_mode": "bar", + "number_format": { + "unit": { + "type": "canonical_unit", + "unit_name": "dollar" + } + } + }, + { + "alias": "List ($)", + "formula": "list", + "number_format": { + "unit": { + "type": "canonical_unit", + "unit_name": "dollar" + } + } + }, + { + "alias": "Requests", + "formula": "req" + } + ], + "sort": { + "count": 500, + "order_by": [ + { + "type": "formula", + "index": 0, + "order": "desc" + } + ] + } + } + ], + "has_search_bar": "auto" + }, + "layout": { + "x": 3, + "y": 0, + "width": 9, + "height": 3 + } + }, + { + "id": 403, + "definition": { + "title": "Cost Distribution by Product and Model", + "requests": [ + { + "queries": [ + { + "data_source": "metrics", + "name": "q", + "query": "sum:claude_enterprise_analytics.cost.amount_usd{$report_date,$org_id,$user_email} by {product,model}" + } + ], + "response_format": "scalar", + "formulas": [ + { + "formula": "q" + } + ] + } + ], + "type": "sunburst", + "legend": { + "type": "automatic" + } + }, + "layout": { + "x": 0, + "y": 3, + "width": 12, + "height": 4 + } + }, + { + "id": 404, + "definition": { + "title": "Cost Breakdown by Token Type (by model & product)", + "type": "query_table", + "requests": [ + { + "queries": [ + { + "data_source": "metrics", + "name": "amount", + "query": "sum:claude_enterprise_analytics.cost.amount_usd{$report_date,$org_id,$user_email} by {model,product}", + "aggregator": "last" + }, + { + "data_source": "metrics", + "name": "list", + "query": "sum:claude_enterprise_analytics.cost.list_amount_usd{$report_date,$org_id,$user_email} by {model,product}", + "aggregator": "last" + }, + { + "data_source": "metrics", + "name": "in", + "query": "sum:claude_enterprise_analytics.tokens.uncached_input{$report_date,$org_id} by {model,product}", + "aggregator": "last" + }, + { + "data_source": "metrics", + "name": "out", + "query": "sum:claude_enterprise_analytics.tokens.output{$report_date,$org_id} by {model,product}", + "aggregator": "last" + }, + { + "data_source": "metrics", + "name": "cr", + "query": "sum:claude_enterprise_analytics.tokens.cache_read{$report_date,$org_id} by {model,product}", + "aggregator": "last" + }, + { + "data_source": "metrics", + "name": "cw1h", + "query": "sum:claude_enterprise_analytics.tokens.cache_write_1h{$report_date,$org_id} by {model,product}", + "aggregator": "last" + }, + { + "data_source": "metrics", + "name": "cw5m", + "query": "sum:claude_enterprise_analytics.tokens.cache_write_5m{$report_date,$org_id} by {model,product}", + "aggregator": "last" + } + ], + "response_format": "scalar", + "sort": { + "count": 500, + "order_by": [ + { + "type": "formula", + "index": 0, + "order": "desc" + } + ] + }, + "formulas": [ + { + "alias": "Cost ($)", + "formula": "amount", + "number_format": { + "unit": { + "type": "canonical_unit", + "unit_name": "dollar" + } + } + }, + { + "alias": "List ($)", + "formula": "list", + "number_format": { + "unit": { + "type": "canonical_unit", + "unit_name": "dollar" + } + } + }, + { + "alias": "Uncached In", + "formula": "in" + }, + { + "alias": "Output", + "formula": "out" + }, + { + "alias": "Cache Read", + "formula": "cr" + }, + { + "alias": "Cache Write (1h)", + "formula": "cw1h" + }, + { + "alias": "Cache Write (5m)", + "formula": "cw5m" + } + ] + } + ] + }, + "layout": { + "x": 0, + "y": 7, + "width": 12, + "height": 3 + } + }, + { + "id": 405, + "definition": { + "title": "Cost Visualization by Model and Product", + "type": "treemap", + "requests": [ + { + "queries": [ + { + "data_source": "metrics", + "name": "q", + "query": "sum:claude_enterprise_analytics.cost.amount_usd{$report_date,$org_id,$user_email} by {model,product}" + } + ], + "response_format": "scalar", + "style": { + "palette": "classic" + }, + "formulas": [ + { + "formula": "q" + } + ] + } + ] + }, + "layout": { + "x": 0, + "y": 10, + "width": 12, + "height": 4 + } + }, + { + "id": 409, + "definition": { + "type": "note", + "content": "**Cost Optimizations**: \nUnderstanding your primary cost drivers helps optimize spending. Input tokens typically cost less than output tokens, while cache reads offer significant savings. Monitor the distribution of costs across different token types to identify optimization opportunities.\n\nImplement prompt optimization to reduce token usage, use appropriate models for each task, use caching for repeated contexts, and [batch high volume requests](https://docs.anthropic.com/en/docs/build-with-claude/batch-processing). Consider a routing layer that selects the most cost-effective model based on task requirements.", + "background_color": "green", + "font_size": "14", + "text_align": "left", + "show_tick": false + }, + "layout": { + "x": 0, + "y": 14, + "width": 12, + "height": 2 + } + } + ] + }, + "layout": { + "x": 0, + "y": 2, + "width": 12, + "height": 17 + } + }, + { + "id": 2, + "definition": { + "title": "Token Usage", + "background_color": "vivid_green", + "show_title": true, + "type": "group", + "layout_type": "ordered", + "widgets": [ + { + "id": 209, + "definition": { + "type": "note", + "content": "**Token Usage Analysis**: Monitor token usage across products (claude_code, chat, claude_in_chrome) to understand surface-specific consumption. This helps budget allocation and identifies high-usage areas that might benefit from optimization. Consider implementing usage quotas or alerts for unusual spikes.\n\nIdentify peak usage periods to optimize resource allocation. Understanding usage patterns helps in capacity planning and can reveal opportunities for load balancing across different products.", + "background_color": "green", + "font_size": "14", + "text_align": "left", + "show_tick": false + }, + "layout": { + "x": 0, + "y": 0, + "width": 3, + "height": 3 + } + }, + { + "id": 202, + "definition": { + "title": "Daily Total Tokens", + "type": "query_value", + "requests": [ + { + "response_format": "scalar", + "queries": [ + { + "data_source": "metrics", + "name": "in", + "query": "sum:claude_enterprise_analytics.tokens.uncached_input{$report_date,$org_id,$model,$product}", + "aggregator": "last" + }, + { + "data_source": "metrics", + "name": "out", + "query": "sum:claude_enterprise_analytics.tokens.output{$report_date,$org_id,$model,$product}", + "aggregator": "last" + }, + { + "data_source": "metrics", + "name": "cr", + "query": "sum:claude_enterprise_analytics.tokens.cache_read{$report_date,$org_id,$model,$product}", + "aggregator": "last" + }, + { + "data_source": "metrics", + "name": "cw1h", + "query": "sum:claude_enterprise_analytics.tokens.cache_write_1h{$report_date,$org_id,$model,$product}", + "aggregator": "last" + }, + { + "data_source": "metrics", + "name": "cw5m", + "query": "sum:claude_enterprise_analytics.tokens.cache_write_5m{$report_date,$org_id,$model,$product}", + "aggregator": "last" + } + ], + "formulas": [ + { + "formula": "in + out + cr + cw1h + cw5m" + } + ] + } + ], + "autoscale": true, + "text_align": "center", + "precision": 0 + }, + "layout": { + "x": 3, + "y": 0, + "width": 3, + "height": 3 + } + }, + { + "id": 201, + "definition": { + "title": "Token Usage by Day", + "title_size": "16", + "title_align": "left", + "show_legend": true, + "legend_layout": "auto", + "legend_columns": [ + "avg", + "min", + "max", + "value", + "sum" + ], + "type": "timeseries", + "requests": [ + { + "response_format": "timeseries", + "queries": [ + { + "data_source": "metrics", + "name": "in", + "query": "sum:claude_enterprise_analytics.tokens.uncached_input{$org_id,$model,$product} by {report_date}", + "aggregator": "last" + } + ], + "formulas": [ + { + "formula": "in", + "alias": "uncached input" + } + ], + "style": { + "palette": "dog_classic", + "order_by": "values", + "line_type": "solid", + "line_width": "normal" + }, + "display_type": "bars" + }, + { + "response_format": "timeseries", + "queries": [ + { + "data_source": "metrics", + "name": "out", + "query": "sum:claude_enterprise_analytics.tokens.output{$org_id,$model,$product} by {report_date}", + "aggregator": "last" + } + ], + "formulas": [ + { + "formula": "out", + "alias": "output" + } + ], + "style": { + "palette": "dog_classic", + "order_by": "values", + "line_type": "solid", + "line_width": "normal" + }, + "display_type": "bars" + }, + { + "response_format": "timeseries", + "queries": [ + { + "data_source": "metrics", + "name": "cr", + "query": "sum:claude_enterprise_analytics.tokens.cache_read{$org_id,$model,$product} by {report_date}", + "aggregator": "last" + } + ], + "formulas": [ + { + "formula": "cr", + "alias": "cache read" + } + ], + "style": { + "palette": "dog_classic", + "order_by": "values", + "line_type": "solid", + "line_width": "normal" + }, + "display_type": "bars" + } + ], + "yaxis": { + "include_zero": true, + "scale": "linear" + } + }, + "layout": { + "x": 6, + "y": 0, + "width": 6, + "height": 3 + } + }, + { + "id": 203, + "definition": { + "title": "Top Token Consumers by Model", + "type": "toplist", + "requests": [ + { + "response_format": "scalar", + "queries": [ + { + "data_source": "metrics", + "name": "q", + "query": "sum:claude_enterprise_analytics.tokens.uncached_input{$report_date,$org_id,$product} by {model}", + "aggregator": "last" + } + ], + "formulas": [ + { + "formula": "q" + } + ], + "sort": { + "count": 10, + "order_by": [ + { + "type": "formula", + "index": 0, + "order": "desc" + } + ] + } + } + ] + }, + "layout": { + "x": 0, + "y": 3, + "width": 6, + "height": 3 + } + }, + { + "id": 204, + "definition": { + "title": "Token Usage Breakdown", + "type": "query_table", + "requests": [ + { + "response_format": "scalar", + "queries": [ + { + "data_source": "metrics", + "name": "in", + "query": "sum:claude_enterprise_analytics.tokens.uncached_input{$report_date,$org_id} by {product,model}", + "aggregator": "last" + }, + { + "data_source": "metrics", + "name": "out", + "query": "sum:claude_enterprise_analytics.tokens.output{$report_date,$org_id} by {product,model}", + "aggregator": "last" + } + ], + "sort": { + "order_by": [ + { + "type": "formula", + "index": 0, + "order": "desc" + } + ], + "count": 500 + }, + "formulas": [ + { + "alias": "Input Tokens", + "formula": "in" + }, + { + "alias": "Output Tokens", + "formula": "out" + } + ] + } + ] + }, + "layout": { + "x": 6, + "y": 3, + "width": 6, + "height": 3 + } + }, + { + "id": 205, + "definition": { + "type": "note", + "content": "**Input vs Output Tokens**: Input tokens are prompts and context sent to Claude; output tokens are the generated responses. Monitor the ratio to optimize prompt engineering and reduce unnecessary context. A high input-to-output ratio might indicate verbose prompts that could be streamlined.", + "background_color": "green", + "font_size": "14", + "text_align": "left", + "show_tick": false + }, + "layout": { + "x": 0, + "y": 6, + "width": 3, + "height": 2 + } + }, + { + "id": 7106833146706646, + "definition": { + "title": "Daily Input Tokens (uncached)", + "type": "query_value", + "requests": [ + { + "response_format": "scalar", + "queries": [ + { + "data_source": "metrics", + "name": "q", + "query": "sum:claude_enterprise_analytics.tokens.uncached_input{$report_date,$org_id,$model,$product}", + "aggregator": "last" + } + ], + "formulas": [ + { + "formula": "q" + } + ] + } + ], + "autoscale": true, + "text_align": "center", + "precision": 0 + }, + "layout": { + "x": 3, + "y": 6, + "width": 4, + "height": 2 + } + }, + { + "id": 8885191409050300, + "definition": { + "title": "Daily Output Tokens", + "type": "query_value", + "requests": [ + { + "response_format": "scalar", + "queries": [ + { + "data_source": "metrics", + "name": "q", + "query": "sum:claude_enterprise_analytics.tokens.output{$report_date,$org_id,$model,$product}", + "aggregator": "last" + } + ], + "formulas": [ + { + "formula": "q" + } + ] + } + ], + "autoscale": true, + "text_align": "center", + "precision": 0 + }, + "layout": { + "x": 7, + "y": 6, + "width": 5, + "height": 2 + } + }, + { + "id": 207, + "definition": { + "type": "note", + "content": "**Token Usage Optimizations**: Reduce token usage by implementing prompt caching for repeated contexts, using system prompts efficiently, and implementing response streaming. Regular monitoring helps identify optimization opportunities.", + "background_color": "green", + "font_size": "14", + "text_align": "left", + "show_tick": false + }, + "layout": { + "x": 0, + "y": 8, + "width": 12, + "height": 1 + } + } + ] + }, + "layout": { + "x": 0, + "y": 19, + "width": 12, + "height": 10 + } + }, + { + "id": 3, + "definition": { + "title": "Cache Performance", + "background_color": "vivid_green", + "show_title": true, + "type": "group", + "layout_type": "ordered", + "widgets": [ + { + "id": 304, + "definition": { + "type": "note", + "content": "**Cache Strategy**: Prompt caching lets you store and reuse context within your prompt. This makes it practical to include additional information \u2014 detailed instructions, example responses \u2014 that improves every response Claude generates.\n\nAnthropic's caching system offers 5-minute and 1-hour ephemeral caches to reduce token costs for repeated context. Improve cache performance by structuring prompts with stable prefixes and grouping similar requests.\n\nFor the best caching strategy, see [Anthropic's caching recommendations](https://docs.anthropic.com/en/docs/build-with-claude/prompt-caching#when-to-use-the-1-hour-cache).", + "background_color": "green", + "font_size": "14", + "text_align": "left", + "show_tick": false + }, + "layout": { + "x": 0, + "y": 0, + "width": 3, + "height": 3 + } + }, + { + "id": 301, + "definition": { + "title": "Cache Token Usage (by day)", + "title_size": "16", + "title_align": "left", + "type": "query_table", + "requests": [ + { + "response_format": "scalar", + "queries": [ + { + "data_source": "metrics", + "name": "cr", + "query": "sum:claude_enterprise_analytics.tokens.cache_read{$report_date,$org_id,$product} by {report_date,model}", + "aggregator": "last" + }, + { + "data_source": "metrics", + "name": "cw1h", + "query": "sum:claude_enterprise_analytics.tokens.cache_write_1h{$report_date,$org_id,$product} by {report_date,model}", + "aggregator": "last" + }, + { + "data_source": "metrics", + "name": "cw5m", + "query": "sum:claude_enterprise_analytics.tokens.cache_write_5m{$report_date,$org_id,$product} by {report_date,model}", + "aggregator": "last" + } + ], + "formulas": [ + { + "alias": "Cache Read", + "formula": "cr", + "cell_display_mode": "bar" + }, + { + "alias": "Cache Write (1h)", + "formula": "cw1h", + "cell_display_mode": "bar" + }, + { + "alias": "Cache Write (5m)", + "formula": "cw5m", + "cell_display_mode": "bar" + } + ], + "sort": { + "count": 500, + "order_by": [ + { + "type": "formula", + "index": 0, + "order": "desc" + } + ] + } + } + ], + "has_search_bar": "auto" + }, + "layout": { + "x": 3, + "y": 0, + "width": 5, + "height": 3 + } + }, + { + "id": 3321592471666638, + "definition": { + "title": "Token Consumption Saved by Cache (by day & model)", + "title_size": "16", + "title_align": "left", + "type": "query_table", + "requests": [ + { + "response_format": "scalar", + "queries": [ + { + "data_source": "metrics", + "name": "cr", + "query": "sum:claude_enterprise_analytics.tokens.cache_read{$report_date,$org_id,$product,$user_email} by {report_date,model}", + "aggregator": "last" + }, + { + "data_source": "metrics", + "name": "cw1h", + "query": "sum:claude_enterprise_analytics.tokens.cache_write_1h{$report_date,$org_id,$product,$user_email} by {report_date,model}", + "aggregator": "last" + }, + { + "data_source": "metrics", + "name": "cw5m", + "query": "sum:claude_enterprise_analytics.tokens.cache_write_5m{$report_date,$org_id,$product,$user_email} by {report_date,model}", + "aggregator": "last" + } + ], + "formulas": [ + { + "alias": "Savings (tokens)", + "formula": "((1.25 * cw5m) + (2 * cw1h) + (0.1 * cr)) - cr", + "cell_display_mode": "bar" + } + ], + "sort": { + "count": 500, + "order_by": [ + { + "type": "formula", + "index": 0, + "order": "desc" + } + ] + } + } + ], + "has_search_bar": "auto" + }, + "layout": { + "x": 8, + "y": 0, + "width": 4, + "height": 3 + } + }, + { + "id": 305, + "definition": { + "type": "note", + "content": "**Cache Hit Rate**: A high cache hit rate significantly reduces costs as cached tokens are charged at reduced rates compared to new input tokens.", + "background_color": "green", + "font_size": "14", + "text_align": "left", + "show_tick": false + }, + "layout": { + "x": 0, + "y": 3, + "width": 3, + "height": 2 + } + }, + { + "id": 302, + "definition": { + "title": "Cache Hit Rate (daily)", + "type": "query_value", + "requests": [ + { + "response_format": "scalar", + "queries": [ + { + "data_source": "metrics", + "name": "cr", + "query": "sum:claude_enterprise_analytics.tokens.cache_read{$report_date,$org_id,$model,$product}", + "aggregator": "last" + }, + { + "data_source": "metrics", + "name": "in", + "query": "sum:claude_enterprise_analytics.tokens.uncached_input{$report_date,$org_id,$model,$product}", + "aggregator": "last" + }, + { + "data_source": "metrics", + "name": "cw1h", + "query": "sum:claude_enterprise_analytics.tokens.cache_write_1h{$report_date,$org_id,$model,$product}", + "aggregator": "last" + }, + { + "data_source": "metrics", + "name": "cw5m", + "query": "sum:claude_enterprise_analytics.tokens.cache_write_5m{$report_date,$org_id,$model,$product}", + "aggregator": "last" + } + ], + "formulas": [ + { + "formula": "(cr / (cr + in + cw1h + cw5m)) * 100", + "number_format": { + "unit": { + "type": "canonical_unit", + "unit_name": "percent" + } + } + } + ] + } + ], + "autoscale": false, + "text_align": "center", + "precision": 2 + }, + "layout": { + "x": 3, + "y": 3, + "width": 3, + "height": 2 + } + }, + { + "id": 5373233506863203, + "definition": { + "title": "Cache Hit Rate (by day & model)", + "title_size": "16", + "title_align": "left", + "type": "query_table", + "requests": [ + { + "response_format": "scalar", + "queries": [ + { + "data_source": "metrics", + "name": "cr", + "query": "sum:claude_enterprise_analytics.tokens.cache_read{$report_date,$org_id,$product} by {report_date,model}", + "aggregator": "last" + }, + { + "data_source": "metrics", + "name": "in", + "query": "sum:claude_enterprise_analytics.tokens.uncached_input{$report_date,$org_id,$product} by {report_date,model}", + "aggregator": "last" + }, + { + "data_source": "metrics", + "name": "cw1h", + "query": "sum:claude_enterprise_analytics.tokens.cache_write_1h{$report_date,$org_id,$product} by {report_date,model}", + "aggregator": "last" + }, + { + "data_source": "metrics", + "name": "cw5m", + "query": "sum:claude_enterprise_analytics.tokens.cache_write_5m{$report_date,$org_id,$product} by {report_date,model}", + "aggregator": "last" + } + ], + "formulas": [ + { + "alias": "Hit %", + "formula": "(cr / (cr + in + cw1h + cw5m)) * 100", + "cell_display_mode": "bar", + "number_format": { + "unit": { + "type": "canonical_unit", + "unit_name": "percent" + } + } + } + ], + "sort": { + "count": 500, + "order_by": [ + { + "type": "formula", + "index": 0, + "order": "desc" + } + ] + } + } + ], + "has_search_bar": "auto" + }, + "layout": { + "x": 6, + "y": 3, + "width": 6, + "height": 2 + } + }, + { + "id": 303, + "definition": { + "title": "Cache Read Volume (by day & model)", + "title_size": "16", + "title_align": "left", + "type": "query_table", + "requests": [ + { + "response_format": "scalar", + "queries": [ + { + "data_source": "metrics", + "name": "cr", + "query": "sum:claude_enterprise_analytics.tokens.cache_read{$report_date,$org_id,$product} by {report_date,model}", + "aggregator": "last" + } + ], + "formulas": [ + { + "alias": "Cache Read Tokens", + "formula": "cr", + "cell_display_mode": "bar" + } + ], + "sort": { + "count": 500, + "order_by": [ + { + "type": "formula", + "index": 0, + "order": "desc" + } + ] + } + } + ], + "has_search_bar": "auto" + }, + "layout": { + "x": 0, + "y": 5, + "width": 6, + "height": 3 + } + }, + { + "id": 4026529261321278, + "definition": { + "title": "Cache Write Breakdown", + "title_size": "16", + "title_align": "left", + "type": "query_table", + "requests": [ + { + "response_format": "scalar", + "queries": [ + { + "data_source": "metrics", + "name": "cw5m", + "query": "sum:claude_enterprise_analytics.tokens.cache_write_5m{$report_date,$org_id,$product} by {model}", + "aggregator": "last" + }, + { + "data_source": "metrics", + "name": "cw1h", + "query": "sum:claude_enterprise_analytics.tokens.cache_write_1h{$report_date,$org_id,$product} by {model}", + "aggregator": "last" + } + ], + "sort": { + "count": 500, + "order_by": [ + { + "type": "formula", + "index": 0, + "order": "desc" + } + ] + }, + "formulas": [ + { + "cell_display_mode": "bar", + "alias": "5m Cache Write", + "formula": "cw5m" + }, + { + "cell_display_mode": "bar", + "alias": "1h Cache Write", + "formula": "cw1h" + } + ] + } + ], + "has_search_bar": "auto" + }, + "layout": { + "x": 6, + "y": 5, + "width": 6, + "height": 3 + } + }, + { + "id": 306, + "definition": { + "type": "note", + "content": "**Caching Cost Impact**: Cache reads cost significantly less than cache writes or regular input tokens. Monitor the ratio of cache reads to writes to assess efficiency. Consider implementing prompt templates that maximize cache reuse, especially for system prompts and common contexts.\n\nThe [Message Batches API](https://docs.anthropic.com/en/docs/build-with-claude/batch-processing#using-prompt-caching-with-message-batches) also supports prompt caching, allowing you to potentially reduce costs and processing time for batch requests. Discounts from prompt caching and Message Batches can stack.", + "background_color": "green", + "font_size": "14", + "text_align": "left", + "show_tick": false + }, + "layout": { + "x": 0, + "y": 8, + "width": 12, + "height": 2 + } + } + ] + }, + "layout": { + "x": 0, + "y": 29, + "width": 12, + "height": 11 + } + }, + { + "id": 5, + "definition": { + "title": "Model Performance", + "background_color": "vivid_green", + "show_title": true, + "type": "group", + "layout_type": "ordered", + "widgets": [ + { + "id": 505, + "definition": { + "type": "note", + "content": "**Model Performance**: Monitor token throughput and request distribution across models. Higher-tier models generally provide better quality but at increased cost. Balance these factors based on your specific use case requirements.\n\nAnalyze how workloads are distributed across models to ensure optimal resource utilization. Consider implementing intelligent routing that automatically selects models based on task complexity, urgency, and budget constraints.", + "background_color": "green", + "font_size": "14", + "text_align": "left", + "show_tick": false + }, + "layout": { + "x": 0, + "y": 0, + "width": 3, + "height": 3 + } + }, + { + "id": 501, + "definition": { + "title": "Token Usage by Day & Model", + "title_size": "16", + "title_align": "left", + "show_legend": true, + "legend_layout": "auto", + "legend_columns": [ + "avg", + "min", + "max", + "value", + "sum" + ], + "type": "timeseries", + "requests": [ + { + "response_format": "timeseries", + "queries": [ + { + "data_source": "metrics", + "name": "in", + "query": "sum:claude_enterprise_analytics.tokens.uncached_input{$org_id,$product,$user_email} by {report_date,model}", + "aggregator": "last" + } + ], + "formulas": [ + { + "formula": "in" + } + ], + "style": { + "palette": "dog_classic", + "order_by": "values", + "line_type": "solid", + "line_width": "normal" + }, + "display_type": "bars" + } + ], + "yaxis": { + "include_zero": true, + "scale": "linear" + } + }, + "layout": { + "x": 3, + "y": 0, + "width": 4, + "height": 3 + } + }, + { + "id": 502, + "definition": { + "title": "Most Used Models", + "type": "toplist", + "requests": [ + { + "response_format": "scalar", + "queries": [ + { + "data_source": "metrics", + "name": "in", + "query": "sum:claude_enterprise_analytics.tokens.uncached_input{$report_date,$org_id,$product,$user_email} by {model}", + "aggregator": "last" + }, + { + "data_source": "metrics", + "name": "out", + "query": "sum:claude_enterprise_analytics.tokens.output{$report_date,$org_id,$product,$user_email} by {model}", + "aggregator": "last" + } + ], + "formulas": [ + { + "formula": "in + out" + } + ], + "sort": { + "count": 10, + "order_by": [ + { + "type": "formula", + "index": 0, + "order": "desc" + } + ] + } + } + ] + }, + "layout": { + "x": 7, + "y": 0, + "width": 5, + "height": 3 + } + }, + { + "id": 508, + "definition": { + "type": "note", + "content": "**Model Optimizations**: Each model has unique strengths. Opus excels at complex reasoning, Sonnet balances capability with efficiency, and Haiku provides rapid responses for straightforward tasks. Tailor prompts and workflows to leverage each model's strengths.\n\nWhen new models are released, test them with representative workloads before full migration. Compare quality, cost, and performance. Implement gradual rollouts and A/B testing to ensure smooth transitions.", + "background_color": "green", + "font_size": "14", + "text_align": "left", + "show_tick": false + }, + "layout": { + "x": 0, + "y": 3, + "width": 3, + "height": 3 + } + }, + { + "id": 503, + "definition": { + "title": "Cost vs Usage Correlation", + "type": "scatterplot", + "requests": { + "table": { + "response_format": "scalar", + "queries": [ + { + "data_source": "metrics", + "name": "cost", + "query": "sum:claude_enterprise_analytics.cost.amount_usd{$report_date,$org_id,$user_email} by {model,product}", + "aggregator": "last" + }, + { + "data_source": "metrics", + "name": "in", + "query": "sum:claude_enterprise_analytics.tokens.uncached_input{$report_date,$org_id,$user_email} by {model,product}", + "aggregator": "last" + }, + { + "data_source": "metrics", + "name": "out", + "query": "sum:claude_enterprise_analytics.tokens.output{$report_date,$org_id,$user_email} by {model,product}", + "aggregator": "last" + } + ], + "formulas": [ + { + "dimension": "x", + "formula": "cost" + }, + { + "dimension": "y", + "number_format": { + "unit": { + "type": "custom_unit_label", + "label": "tokens" + } + }, + "formula": "in + out" + } + ] + } + }, + "xaxis": { + "scale": "linear", + "include_zero": true, + "min": "auto", + "max": "auto" + }, + "yaxis": { + "scale": "linear", + "include_zero": true, + "min": "auto", + "max": "auto" + }, + "color_by_groups": [] + }, + "layout": { + "x": 3, + "y": 3, + "width": 4, + "height": 3 + } + }, + { + "id": 3009394555165132, + "definition": { + "title": "Cost and Token Usage (by day & model)", + "title_size": "16", + "title_align": "left", + "type": "query_table", + "requests": [ + { + "response_format": "scalar", + "queries": [ + { + "data_source": "metrics", + "name": "cost", + "query": "sum:claude_enterprise_analytics.cost.amount_usd{$report_date,$org_id,$product,$user_email} by {report_date,model}", + "aggregator": "last" + }, + { + "data_source": "metrics", + "name": "in", + "query": "sum:claude_enterprise_analytics.tokens.uncached_input{$report_date,$org_id,$product,$user_email} by {report_date,model}", + "aggregator": "last" + }, + { + "data_source": "metrics", + "name": "out", + "query": "sum:claude_enterprise_analytics.tokens.output{$report_date,$org_id,$product,$user_email} by {report_date,model}", + "aggregator": "last" + } + ], + "formulas": [ + { + "alias": "Cost ($)", + "formula": "cost", + "cell_display_mode": "bar", + "number_format": { + "unit": { + "type": "canonical_unit", + "unit_name": "dollar" + } + } + }, + { + "alias": "Input Tokens", + "formula": "in" + }, + { + "alias": "Output Tokens", + "formula": "out" + } + ], + "sort": { + "count": 500, + "order_by": [ + { + "type": "formula", + "index": 0, + "order": "desc" + } + ] + } + } + ], + "has_search_bar": "auto" + }, + "layout": { + "x": 7, + "y": 3, + "width": 5, + "height": 3 + } + } + ] + }, + "layout": { + "x": 0, + "y": 40, + "width": 12, + "height": 7 + } + }, + { + "id": 7, + "definition": { + "title": "User Analytics", + "background_color": "vivid_green", + "show_title": true, + "type": "group", + "layout_type": "ordered", + "widgets": [ + { + "id": 710, + "definition": { + "type": "note", + "content": "**Usage Anomaly Detection**: Monitor for unusual user activity patterns that might indicate inefficient usage, runaway processes, or potential security concerns. Set up alerts on `claude_enterprise_analytics.user.cost.amount_usd` for users exceeding normal consumption thresholds.", + "background_color": "green", + "font_size": "14", + "text_align": "left", + "show_tick": false + }, + "layout": { + "x": 0, + "y": 0, + "width": 3, + "height": 3 + } + }, + { + "id": 701, + "definition": { + "title": "Token Usage by Day & User", + "title_size": "16", + "title_align": "left", + "show_legend": true, + "legend_layout": "auto", + "legend_columns": [ + "avg", + "min", + "max", + "value", + "sum" + ], + "type": "timeseries", + "requests": [ + { + "response_format": "timeseries", + "queries": [ + { + "data_source": "metrics", + "name": "tot", + "query": "sum:claude_enterprise_analytics.user.tokens_total{$org_id} by {report_date,user_email}", + "aggregator": "last" + } + ], + "formulas": [ + { + "formula": "tot", + "alias": "tokens" + } + ], + "style": { + "palette": "dog_classic", + "order_by": "values", + "line_type": "solid", + "line_width": "normal" + }, + "display_type": "bars" + } + ], + "yaxis": { + "include_zero": true, + "scale": "linear" + } + }, + "layout": { + "x": 3, + "y": 0, + "width": 9, + "height": 3 + } + }, + { + "id": 708, + "definition": { + "type": "note", + "content": "**User Analytics**: Monitor individual user adoption and usage patterns to identify power users, training needs, and potential optimization opportunities. High-volume users may benefit from training on efficient prompt engineering to reduce token consumption.", + "background_color": "green", + "font_size": "14", + "text_align": "left", + "show_tick": false + }, + "layout": { + "x": 0, + "y": 3, + "width": 3, + "height": 3 + } + }, + { + "id": 705, + "definition": { + "title": "User Activity by Day", + "title_size": "16", + "title_align": "left", + "show_legend": true, + "legend_layout": "auto", + "legend_columns": [ + "avg", + "min", + "max", + "value", + "sum" + ], + "type": "timeseries", + "requests": [ + { + "response_format": "timeseries", + "queries": [ + { + "data_source": "metrics", + "name": "tot", + "query": "sum:claude_enterprise_analytics.user.tokens_total{$org_id} by {report_date,user_email}", + "aggregator": "last" + } + ], + "formulas": [ + { + "formula": "tot", + "alias": "tokens by user" + } + ], + "style": { + "palette": "dog_classic", + "order_by": "values", + "line_type": "solid", + "line_width": "normal" + }, + "display_type": "bars" + } + ], + "yaxis": { + "include_zero": true, + "scale": "linear" + } + }, + "layout": { + "x": 3, + "y": 3, + "width": 9, + "height": 3 + } + }, + { + "id": 709, + "definition": { + "type": "note", + "content": "**User Token Attribution**: Track costs by individual users to enable chargeback models and budget allocation. This data helps surface high-usage individuals and encourages responsible usage. Consider implementing user-level alerts for unusual consumption patterns.", + "background_color": "green", + "font_size": "14", + "text_align": "left", + "show_tick": false + }, + "layout": { + "x": 0, + "y": 6, + "width": 3, + "height": 3 + } + }, + { + "id": 702, + "definition": { + "title": "Top Users by Token Consumption", + "type": "toplist", + "requests": [ + { + "response_format": "scalar", + "queries": [ + { + "data_source": "metrics", + "name": "q", + "query": "sum:claude_enterprise_analytics.user.tokens_total{$report_date,$org_id,user_email:*,!user_email:unknown} by {user_email}", + "aggregator": "last" + } + ], + "formulas": [ + { + "formula": "q" + } + ], + "sort": { + "count": 100, + "order_by": [ + { + "type": "formula", + "index": 0, + "order": "desc" + } + ] + } + } + ] + }, + "layout": { + "x": 3, + "y": 6, + "width": 3, + "height": 3 + } + }, + { + "id": 704, + "definition": { + "title": "User Usage Breakdown", + "type": "query_table", + "requests": [ + { + "response_format": "scalar", + "queries": [ + { + "data_source": "metrics", + "name": "in", + "query": "sum:claude_enterprise_analytics.user.uncached_input_tokens{$report_date,$org_id} by {user_email}", + "aggregator": "last" + }, + { + "data_source": "metrics", + "name": "out", + "query": "sum:claude_enterprise_analytics.user.output_tokens{$report_date,$org_id} by {user_email}", + "aggregator": "last" + }, + { + "data_source": "metrics", + "name": "cr", + "query": "sum:claude_enterprise_analytics.user.cache_read_tokens{$report_date,$org_id} by {user_email}", + "aggregator": "last" + }, + { + "data_source": "metrics", + "name": "cost", + "query": "sum:claude_enterprise_analytics.user.cost.amount_usd{$report_date,$org_id} by {user_email}", + "aggregator": "last" + } + ], + "sort": { + "count": 500, + "order_by": [ + { + "type": "formula", + "index": 3, + "order": "desc" + } + ] + }, + "formulas": [ + { + "alias": "Input Tokens", + "formula": "in" + }, + { + "alias": "Output Tokens", + "formula": "out" + }, + { + "alias": "Cached Input Tokens", + "formula": "cr" + }, + { + "alias": "Cost ($)", + "formula": "cost", + "number_format": { + "unit": { + "type": "canonical_unit", + "unit_name": "dollar" + } + } + } + ] + } + ] + }, + "layout": { + "x": 6, + "y": 6, + "width": 6, + "height": 3 + } + }, + { + "id": 706, + "definition": { + "title": "User Cost Distribution by Product", + "requests": [ + { + "queries": [ + { + "name": "q", + "data_source": "metrics", + "query": "sum:claude_enterprise_analytics.user.cost.amount_usd{$report_date,$org_id} by {user_email}", + "aggregator": "last" + } + ], + "response_format": "scalar", + "style": { + "palette": "dog_classic" + }, + "formulas": [ + { + "formula": "q" + } + ], + "sort": { + "order_by": [ + { + "type": "formula", + "index": 0, + "order": "desc" + } + ], + "count": 500 + } + } + ], + "type": "sunburst", + "legend": { + "type": "automatic" + } + }, + "layout": { + "x": 0, + "y": 9, + "width": 5, + "height": 4 + } + }, + { + "id": 707, + "definition": { + "title": "User Usage Visualization", + "type": "treemap", + "requests": [ + { + "queries": [ + { + "name": "q", + "data_source": "metrics", + "query": "sum:claude_enterprise_analytics.user.tokens_total{$report_date,$org_id} by {user_email}", + "aggregator": "last" + } + ], + "response_format": "scalar", + "style": { + "palette": "dog_classic" + }, + "formulas": [ + { + "formula": "q" + } + ] + } + ] + }, + "layout": { + "x": 5, + "y": 9, + "width": 7, + "height": 4 + } + }, + { + "id": 7960539772614678, + "definition": { + "title": "Cache Usage by User", + "title_size": "16", + "title_align": "left", + "type": "query_table", + "requests": [ + { + "queries": [ + { + "data_source": "metrics", + "name": "cr", + "query": "sum:claude_enterprise_analytics.user.cache_read_tokens{$report_date,$org_id} by {user_email}", + "aggregator": "last" + }, + { + "data_source": "metrics", + "name": "cw5m", + "query": "sum:claude_enterprise_analytics.user.cache_write_5m_tokens{$report_date,$org_id} by {user_email}", + "aggregator": "last" + }, + { + "data_source": "metrics", + "name": "cw1h", + "query": "sum:claude_enterprise_analytics.user.cache_write_1h_tokens{$report_date,$org_id} by {user_email}", + "aggregator": "last" + }, + { + "data_source": "metrics", + "name": "in", + "query": "sum:claude_enterprise_analytics.user.uncached_input_tokens{$report_date,$org_id} by {user_email}", + "aggregator": "last" + }, + { + "data_source": "metrics", + "name": "out", + "query": "sum:claude_enterprise_analytics.user.output_tokens{$report_date,$org_id} by {user_email}", + "aggregator": "last" + } + ], + "response_format": "scalar", + "sort": { + "count": 500, + "order_by": [ + { + "type": "formula", + "index": 0, + "order": "desc" + } + ] + }, + "formulas": [ + { + "alias": "Cache Read", + "cell_display_mode": "bar", + "formula": "cr" + }, + { + "alias": "Cache Write (5m)", + "cell_display_mode": "bar", + "formula": "cw5m" + }, + { + "alias": "Cache Write (1h)", + "cell_display_mode": "bar", + "formula": "cw1h" + }, + { + "alias": "Input Tokens", + "formula": "in" + }, + { + "alias": "Output Tokens", + "formula": "out" + }, + { + "cell_display_mode": "bar", + "alias": "Cache Hit %", + "formula": "(cr / (cr + cw5m + cw1h + in)) * 100" + } + ] + } + ], + "has_search_bar": "auto" + }, + "layout": { + "x": 0, + "y": 13, + "width": 12, + "height": 4 + } + }, + { + "id": 712, + "definition": { + "type": "note", + "content": "**Usage Optimizations**: Track individual user patterns to identify friction points and optimization opportunities. Users with excessive token usage may benefit from additional training or access to different models better suited to their tasks.\n\nAnalyze usage patterns to segment users by consumption levels, model preferences, and use cases. This segmentation helps tailor training programs, optimize model allocation, and identify opportunities for cost containment.", + "background_color": "green", + "font_size": "14", + "text_align": "left", + "show_tick": false + }, + "layout": { + "x": 0, + "y": 17, + "width": 12, + "height": 2 + } + } + ] + }, + "layout": { + "x": 0, + "y": 47, + "width": 12, + "height": 20 + } + }, + { + "id": 6, + "definition": { + "title": "Web Usage", + "background_color": "vivid_green", + "show_title": true, + "type": "group", + "layout_type": "ordered", + "widgets": [ + { + "id": 604, + "definition": { + "type": "note", + "content": "**Web Usage**: The [Anthropic web search tool](https://docs.anthropic.com/en/docs/agents-and-tools/tool-use/web-search-tool) gives Claude direct access to real-time web content beyond its knowledge cutoff. Monitor usage to understand how often real-time information is requested. High usage might indicate opportunities for leveraging [Anthropic's prompt caching feature](https://docs.anthropic.com/en/docs/agents-and-tools/tool-use/web-search-tool#prompt-caching).", + "background_color": "green", + "font_size": "14", + "text_align": "left", + "show_tick": false + }, + "layout": { + "x": 0, + "y": 0, + "width": 4, + "height": 2 + } + }, + { + "id": 607, + "definition": { + "type": "note", + "content": "**Search Pattern Analysis**: Analyze common search patterns to identify frequently needed information that could be pre-loaded or cached. Understanding search patterns helps optimize knowledge management strategies and can reduce reliance on real-time searches for static information.", + "background_color": "green", + "font_size": "14", + "text_align": "left", + "show_tick": false + }, + "layout": { + "x": 4, + "y": 0, + "width": 8, + "height": 2 + } + }, + { + "id": 602, + "definition": { + "title": "Daily Web Searches", + "type": "query_value", + "requests": [ + { + "response_format": "scalar", + "queries": [ + { + "data_source": "metrics", + "name": "q", + "query": "sum:claude_enterprise_analytics.web_search_requests{$report_date,$org_id,$model,$product}", + "aggregator": "last" + } + ], + "formulas": [ + { + "formula": "q" + } + ] + } + ], + "autoscale": true, + "text_align": "center", + "precision": 0 + }, + "layout": { + "x": 0, + "y": 2, + "width": 4, + "height": 3 + } + }, + { + "id": 601, + "definition": { + "title": "Web Search by Day & Model", + "title_size": "16", + "title_align": "left", + "show_legend": true, + "legend_layout": "auto", + "legend_columns": [ + "avg", + "min", + "max", + "value", + "sum" + ], + "type": "timeseries", + "requests": [ + { + "response_format": "timeseries", + "queries": [ + { + "data_source": "metrics", + "name": "ws", + "query": "sum:claude_enterprise_analytics.web_search_requests{$org_id,$product} by {report_date,model}", + "aggregator": "last" + } + ], + "formulas": [ + { + "formula": "ws" + } + ], + "style": { + "palette": "dog_classic", + "order_by": "values", + "line_type": "solid", + "line_width": "normal" + }, + "display_type": "bars" + } + ], + "yaxis": { + "include_zero": true, + "scale": "linear" + } + }, + "layout": { + "x": 4, + "y": 2, + "width": 8, + "height": 3 + } + }, + { + "id": 603, + "definition": { + "title": "Web Search Usage by Product", + "type": "query_table", + "requests": [ + { + "queries": [ + { + "data_source": "metrics", + "name": "q", + "query": "sum:claude_enterprise_analytics.web_search_requests{$report_date,$org_id,$model} by {product}", + "aggregator": "last" + } + ], + "response_format": "scalar", + "sort": { + "count": 10, + "order_by": [ + { + "type": "formula", + "index": 0, + "order": "desc" + } + ] + }, + "formulas": [ + { + "formula": "q", + "cell_display_mode": "bar" + } + ] + } + ], + "has_search_bar": "auto" + }, + "layout": { + "x": 0, + "y": 5, + "width": 12, + "height": 3 + } + }, + { + "id": 606, + "definition": { + "type": "note", + "content": "**Web Usage Cost Impact**: Web searches can be token-intensive. Monitor the cost-benefit ratio of search-enabled responses versus standard completions. Consider implementing search result summaries or excerpts to reduce the token cost of incorporating search results.", + "background_color": "green", + "font_size": "14", + "text_align": "left", + "show_tick": false + }, + "layout": { + "x": 0, + "y": 8, + "width": 12, + "height": 1 + } + } + ] + }, + "layout": { + "x": 0, + "y": 67, + "width": 12, + "height": 10 + } + } + ], + "template_variables": [ + { + "name": "report_date", + "prefix": "report_date", + "available_values": [], + "default": "*" + }, + { + "name": "model", + "prefix": "model", + "available_values": [], + "default": "*" + }, + { + "name": "user_email", + "prefix": "user_email", + "available_values": [], + "default": "*" + }, + { + "name": "product", + "prefix": "product", + "available_values": [], + "default": "*" + }, + { + "name": "org_id", + "prefix": "org_id", + "available_values": [], + "default": "*" + } + ], + "layout_type": "ordered", + "notify_list": [], + "reflow_type": "fixed" +} diff --git a/claude_enterprise_analytics/assets/service_checks.json b/claude_enterprise_analytics/assets/service_checks.json new file mode 100644 index 0000000000..10401f60d1 --- /dev/null +++ b/claude_enterprise_analytics/assets/service_checks.json @@ -0,0 +1,11 @@ +[ + { + "agent_version": "7.50.0", + "integration": "Claude Enterprise Analytics", + "check": "claude_enterprise_analytics.can_connect", + "statuses": ["ok", "critical"], + "groups": [], + "name": "Claude Enterprise Analytics API reachable", + "description": "Returns CRITICAL if the Anthropic Analytics API could not be reached or returned a non-success status during the check run, otherwise returns OK." + } +] diff --git a/claude_enterprise_analytics/changelog.d/1.added b/claude_enterprise_analytics/changelog.d/1.added new file mode 100644 index 0000000000..aa949b47b7 --- /dev/null +++ b/claude_enterprise_analytics/changelog.d/1.added @@ -0,0 +1 @@ +Initial Release \ No newline at end of file diff --git a/claude_enterprise_analytics/datadog_checks/claude_enterprise_analytics/__about__.py b/claude_enterprise_analytics/datadog_checks/claude_enterprise_analytics/__about__.py new file mode 100644 index 0000000000..e50f43adfb --- /dev/null +++ b/claude_enterprise_analytics/datadog_checks/claude_enterprise_analytics/__about__.py @@ -0,0 +1,4 @@ +# (C) Datadog, Inc. 2026-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) +__version__ = '0.0.1' diff --git a/claude_enterprise_analytics/datadog_checks/claude_enterprise_analytics/__init__.py b/claude_enterprise_analytics/datadog_checks/claude_enterprise_analytics/__init__.py new file mode 100644 index 0000000000..164cf30e98 --- /dev/null +++ b/claude_enterprise_analytics/datadog_checks/claude_enterprise_analytics/__init__.py @@ -0,0 +1,7 @@ +# (C) Datadog, Inc. 2026-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) +from .__about__ import __version__ +from .check import ClaudeEnterpriseAnalyticsCheck + +__all__ = ['__version__', 'ClaudeEnterpriseAnalyticsCheck'] diff --git a/claude_enterprise_analytics/datadog_checks/claude_enterprise_analytics/_anthropic_client.py b/claude_enterprise_analytics/datadog_checks/claude_enterprise_analytics/_anthropic_client.py new file mode 100644 index 0000000000..64a39134e2 --- /dev/null +++ b/claude_enterprise_analytics/datadog_checks/claude_enterprise_analytics/_anthropic_client.py @@ -0,0 +1,177 @@ +# (C) Typeform Platform 2026-present +# Unofficial integration. Not affiliated with Anthropic, PBC. +# Licensed under a 3-clause BSD style license (see LICENSE) +"""Thin client for the Anthropic Claude Enterprise Analytics API. + +Docs: https://support.claude.com/en/articles/13703965 +Base: https://api.anthropic.com/v1/organizations/analytics + +Endpoint quirks (verified empirically 2026-05-21): +- `/summaries` takes `starting_date` / `ending_date` (calendar dates, + strict-before). Response: {"summaries": [...]}. +- `/users` takes `date` (single day). Response: + {"data": [...], "next_page": "..."}. +- `/usage_report`, take `starting_at` / `ending_at` (RFC3339 timestamps). + `/cost_report`, Response: {"data": [{"results": [...]}], "has_more", "next_page"}. +- `/user_usage_report`, take `starting_at` / `ending_at`. Response: flat + `/user_cost_report` {"data": [...rows w/ actor], "has_more", "next_page"}. + +Pagination cursor token comes back as `next_page` and is sent as `?page=`. +Uses the AgentCheck's `self.http` (a `requests`-compatible session) for transport +so it inherits proxy/TLS/auth configuration from the Datadog Agent. +""" + +from datetime import datetime, timedelta, timezone +from datetime import time as dtime + +ANTHROPIC_API_BASE = "https://api.anthropic.com/v1/organizations/analytics" + +_RETRYABLE_STATUSES = {429, 500, 502, 503, 504} +_MAX_RETRIES = 5 +_BASE_BACKOFF_SECONDS = 1.0 +_PAGE_SIZE = 1000 + + +class AnthropicAnalyticsClient(object): + """Synchronous client used inside the Agent check. + + The check passes in `self.http` (the AgentCheck-managed `requests.Session`) + so we get its retries-around-this-layer behavior + DD Agent proxy support + for free. + """ + + def __init__(self, http, api_key, log, timeout=30.0): + self._http = http + self._api_key = api_key + self._log = log + self._timeout = timeout + + def _headers(self): + return {"x-api-key": self._api_key, "accept": "application/json"} + + def _get(self, path, params): + # `self.http` already wraps connection retries; we add an extra layer + # here only for 429/5xx with backoff, since those are application-level. + import time as _time + + url = "{}{}".format(ANTHROPIC_API_BASE, path) + last_resp = None + for attempt in range(1, _MAX_RETRIES + 1): + resp = self._http.get(url, params=params, headers=self._headers(), timeout=self._timeout) + last_resp = resp + if resp.status_code in _RETRYABLE_STATUSES: + retry_after = resp.headers.get("retry-after") + wait = ( + float(retry_after) + if retry_after and retry_after.replace(".", "", 1).isdigit() + else _BASE_BACKOFF_SECONDS * (2 ** (attempt - 1)) + ) + self._log.warning( + "Anthropic %s -> %d (attempt %d/%d), sleeping %.1fs", + path, + resp.status_code, + attempt, + _MAX_RETRIES, + wait, + ) + _time.sleep(wait) + continue + if resp.status_code >= 400: + self._log.error("Anthropic %s -> %d: %s", path, resp.status_code, resp.text[:500]) + resp.raise_for_status() + return resp.json() + last_resp.raise_for_status() + return last_resp.json() + + def _paginate_rows(self, path, params, rows_key="data"): + page_params = dict(params) + page_params["limit"] = _PAGE_SIZE + while True: + payload = self._get(path, page_params) + for row in payload.get(rows_key) or []: + yield row + token = payload.get("next_page") + has_more = payload.get("has_more") + if not token or has_more is False: + return + page_params = dict(params) + page_params["limit"] = _PAGE_SIZE + page_params["page"] = token + + @staticmethod + def _iso_date(d): + return d.isoformat() + + @staticmethod + def _iso_ts(d): + return datetime.combine(d, dtime.min, tzinfo=timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") + + @staticmethod + def _next_day(d): + return d + timedelta(days=1) + + # --- endpoint wrappers --- + + def summaries(self, day): + payload = self._get( + "/summaries", + { + "starting_date": self._iso_date(day), + "ending_date": self._iso_date(self._next_day(day)), + }, + ) + rows = payload.get("summaries") or [] + return rows[0] if rows else None + + def users(self, day): + for row in self._paginate_rows("/users", {"date": self._iso_date(day)}): + yield row + + def usage_report(self, day, group_by=None): + for row in self._iter_report("/usage_report", day, group_by or ["model", "product"]): + yield row + + def cost_report(self, day, group_by=None): + for row in self._iter_report("/cost_report", day, group_by or ["model", "product"]): + yield row + + def user_usage_report(self, day): + params = { + "starting_at": self._iso_ts(day), + "ending_at": self._iso_ts(self._next_day(day)), + } + for row in self._paginate_rows("/user_usage_report", params): + yield row + + def user_cost_report(self, day): + params = { + "starting_at": self._iso_ts(day), + "ending_at": self._iso_ts(self._next_day(day)), + } + for row in self._paginate_rows("/user_cost_report", params): + yield row + + def _iter_report(self, path, day, group_by): + # /usage_report and /cost_report wrap rows under data[].results[]. + params = { + "starting_at": self._iso_ts(day), + "ending_at": self._iso_ts(self._next_day(day)), + } + gb = [] + for key in group_by: + gb.append(key) + if gb: + params["group_by[]"] = gb + + page_params = dict(params) + while True: + payload = self._get(path, page_params) + for outer in payload.get("data") or []: + for row in outer.get("results") or []: + yield row + token = payload.get("next_page") + has_more = payload.get("has_more") + if not token or has_more is False: + return + page_params = dict(params) + page_params["page"] = token diff --git a/claude_enterprise_analytics/datadog_checks/claude_enterprise_analytics/_mappers.py b/claude_enterprise_analytics/datadog_checks/claude_enterprise_analytics/_mappers.py new file mode 100644 index 0000000000..59090a0749 --- /dev/null +++ b/claude_enterprise_analytics/datadog_checks/claude_enterprise_analytics/_mappers.py @@ -0,0 +1,189 @@ +# (C) Typeform Platform 2026-present +# Unofficial integration. Not affiliated with Anthropic, PBC. +# Licensed under a 3-clause BSD style license (see LICENSE) +"""Anthropic Analytics rows -> Datadog gauge submissions. + +Each mapper is a generator that yields `(metric_name, value, extra_tags_list)` +tuples. The Check iterates over these and calls `self.gauge(name, value, +tags=tags)`. + +Field paths verified empirically against the live API on 2026-05-21. + +Key API quirk: cost amounts come back as STRINGS in USD cents +("131309.570280" → $1,313.10), so we divide by 100 via _cents_to_usd(). +""" + + +# --- helpers --------------------------------------------------------------- + + +def _num(value): + if value is None or value == "": + return 0.0 + try: + return float(value) + except (TypeError, ValueError): + return 0.0 + + +def _cents_to_usd(value): + """Cost-report `amount`/`list_amount` are returned in USD cents despite + the `currency: USD` field. Verified by cross-checking the org-level web + spend dashboard on 2026-05-21.""" + return _num(value) / 100.0 + + +def _g(obj, *path, **kwargs): + default = kwargs.get("default", 0) + cur = obj + for key in path: + if not isinstance(cur, dict): + return default + cur = cur.get(key) + if cur is None: + return default + return cur + + +def _tag(key, value): + if value is None or value == "": + return "{}:unknown".format(key) + return "{}:{}".format(key, value) + + +def _rd_tag(report_date): + return "report_date:{}".format(report_date.isoformat()) + + +# --- /summaries ----------------------------------------------------------- + + +def from_summaries(row, report_date): + if not row: + return + rd = _rd_tag(report_date) + yield ("org.dau", _num(row.get("daily_active_user_count")), [rd]) + yield ("org.wau", _num(row.get("weekly_active_user_count")), [rd]) + yield ("org.mau", _num(row.get("monthly_active_user_count")), [rd]) + yield ("org.seats_assigned", _num(row.get("assigned_seat_count")), [rd]) + yield ("org.invites_pending", _num(row.get("pending_invite_count")), [rd]) + yield ("org.adoption_rate.daily", _num(row.get("daily_adoption_rate")), [rd]) + yield ("org.adoption_rate.weekly", _num(row.get("weekly_adoption_rate")), [rd]) + yield ("org.adoption_rate.monthly", _num(row.get("monthly_adoption_rate")), [rd]) + yield ("org.cowork.dau", _num(row.get("cowork_daily_active_user_count")), [rd]) + yield ("org.cowork.wau", _num(row.get("cowork_weekly_active_user_count")), [rd]) + yield ("org.cowork.mau", _num(row.get("cowork_monthly_active_user_count")), [rd]) + + +# --- /users --------------------------------------------------------------- + + +def from_users(rows, report_date): + rd = _rd_tag(report_date) + for row in rows: + email = _g(row, "user", "email_address", default="unknown") + utags = [rd, _tag("user_email", email)] + yield ("user.chat.messages", _g(row, "chat_metrics", "message_count"), utags) + yield ("user.chat.conversations", _g(row, "chat_metrics", "distinct_conversation_count"), utags) + yield ("user.chat.thinking_messages", _g(row, "chat_metrics", "thinking_message_count"), utags) + yield ("user.projects.used", _g(row, "chat_metrics", "distinct_projects_used_count"), utags) + yield ("user.projects.created", _g(row, "chat_metrics", "distinct_projects_created_count"), utags) + yield ("user.artifacts.created", _g(row, "chat_metrics", "distinct_artifacts_created_count"), utags) + yield ("user.skills.used", _g(row, "chat_metrics", "distinct_skills_used_count"), utags) + yield ("user.connectors.used", _g(row, "chat_metrics", "connectors_used_count"), utags) + yield ("user.files.uploaded", _g(row, "chat_metrics", "distinct_files_uploaded_count"), utags) + + cc_core = _g(row, "claude_code_metrics", "core_metrics", default={}) or {} + yield ("user.claude_code.sessions", _g(cc_core, "distinct_session_count"), utags) + yield ("user.claude_code.commits", _g(cc_core, "commit_count"), utags) + yield ("user.claude_code.prs", _g(cc_core, "pull_request_count"), utags) + yield ("user.claude_code.lines_added", _g(cc_core, "lines_of_code", "added_count"), utags) + yield ("user.claude_code.lines_removed", _g(cc_core, "lines_of_code", "removed_count"), utags) + + tool_actions = _g(row, "claude_code_metrics", "tool_actions", default={}) or {} + for tool_name, tool_data in tool_actions.items(): + if not isinstance(tool_data, dict): + continue + tname = tool_name[: -len("_tool")] if tool_name.endswith("_tool") else tool_name + for outcome in ("accepted", "rejected"): + yield ( + "user.claude_code.tool_actions", + _num(tool_data.get(outcome + "_count")), + utags + [_tag("tool", tname), _tag("outcome", outcome)], + ) + + yield ("user.web_search_count", _num(row.get("web_search_count")), utags) + + +# --- /usage_report (grouped by model, product) ---------------------------- + + +def from_usage_report(rows, report_date): + rd = _rd_tag(report_date) + for row in rows: + tags = [ + rd, + _tag("model", row.get("model")), + _tag("product", row.get("product")), + _tag("context_window", row.get("context_window")), + ] + yield ("tokens.uncached_input", _num(row.get("uncached_input_tokens")), tags) + yield ("tokens.cache_read", _num(row.get("cache_read_input_tokens")), tags) + yield ("tokens.cache_write_1h", _num(_g(row, "cache_creation", "ephemeral_1h_input_tokens")), tags) + yield ("tokens.cache_write_5m", _num(_g(row, "cache_creation", "ephemeral_5m_input_tokens")), tags) + yield ("tokens.output", _num(row.get("output_tokens")), tags) + yield ("requests", _num(row.get("requests")), tags) + yield ("web_search_requests", _num(_g(row, "server_tool_use", "web_search_requests")), tags) + + +# --- /cost_report (grouped by model, product) ----------------------------- + + +def from_cost_report(rows, report_date): + rd = _rd_tag(report_date) + for row in rows: + tags = [ + rd, + _tag("model", row.get("model")), + _tag("product", row.get("product")), + _tag("currency", row.get("currency") or "USD"), + ] + yield ("cost.amount_usd", _cents_to_usd(row.get("amount")), tags) + yield ("cost.list_amount_usd", _cents_to_usd(row.get("list_amount")), tags) + + +# --- /user_usage_report --------------------------------------------------- + + +def from_user_usage_report(rows, report_date): + rd = _rd_tag(report_date) + for row in rows: + email = _g(row, "actor", "email", default="unknown") + utags = [rd, _tag("user_email", email)] + yield ("user.tokens_total", _num(row.get("total_tokens")), utags) + yield ("user.uncached_input_tokens", _num(row.get("uncached_input_tokens")), utags) + yield ("user.output_tokens", _num(row.get("output_tokens")), utags) + yield ("user.cache_read_tokens", _num(row.get("cache_read_input_tokens")), utags) + yield ( + "user.cache_write_5m_tokens", + _num(_g(row, "cache_creation", "ephemeral_5m_input_tokens")), + utags, + ) + yield ( + "user.cache_write_1h_tokens", + _num(_g(row, "cache_creation", "ephemeral_1h_input_tokens")), + utags, + ) + yield ("user.requests", _num(row.get("requests")), utags) + + +# --- /user_cost_report ---------------------------------------------------- + + +def from_user_cost_report(rows, report_date): + rd = _rd_tag(report_date) + for row in rows: + email = _g(row, "actor", "email", default="unknown") + utags = [rd, _tag("user_email", email), _tag("currency", row.get("currency") or "USD")] + yield ("user.cost.amount_usd", _cents_to_usd(row.get("amount")), utags) + yield ("user.cost.list_amount_usd", _cents_to_usd(row.get("list_amount")), utags) diff --git a/claude_enterprise_analytics/datadog_checks/claude_enterprise_analytics/check.py b/claude_enterprise_analytics/datadog_checks/claude_enterprise_analytics/check.py new file mode 100644 index 0000000000..a5100c2b63 --- /dev/null +++ b/claude_enterprise_analytics/datadog_checks/claude_enterprise_analytics/check.py @@ -0,0 +1,97 @@ +# (C) Typeform Platform 2026-present +# Unofficial integration. Not affiliated with Anthropic, PBC. +# Licensed under a 3-clause BSD style license (see LICENSE) +"""Datadog Agent Check for the Anthropic Claude Enterprise Analytics API. + +Polls the six in-scope endpoints (summaries, users, usage_report, cost_report, +user_usage_report, user_cost_report) for a single report-date (default: today +minus the configured lag, which Anthropic itself sets at 3 days), maps each +row to gauges, and submits via `self.gauge()`. + +Architectural note: Anthropic exposes only daily aggregates with a 3-day lag. +Every metric is submitted at agent wall-clock time and tagged with +`report_date:YYYY-MM-DD` representing the activity day. Dashboards must filter +on the `report_date` tag rather than rely on Datadog's wall-clock x-axis. +""" + +from datetime import date, timedelta +from typing import Any # noqa: F401 + +from datadog_checks.base import AgentCheck, ConfigurationError +from datadog_checks.claude_enterprise_analytics import _mappers as mappers +from datadog_checks.claude_enterprise_analytics._anthropic_client import ( + AnthropicAnalyticsClient, +) + +_SERVICE_CHECK_API = "can_connect" +_DEFAULT_LAG_DAYS = 3 + + +class ClaudeEnterpriseAnalyticsCheck(AgentCheck): + __NAMESPACE__ = "claude_enterprise_analytics" + + def __init__(self, name, init_config, instances): + super(ClaudeEnterpriseAnalyticsCheck, self).__init__(name, init_config, instances) + + self._api_key = self.instance.get("anthropic_api_key") + self._org_id = self.instance.get("org_id", "unknown") + self._lag_days = int(self.instance.get("lag_days", _DEFAULT_LAG_DAYS)) + + if not self._api_key: + raise ConfigurationError( + "anthropic_api_key is required. Generate one at claude.ai/analytics/api-keys " + "with the `read:analytics` scope." + ) + + def check(self, _instance): + report_date = date.today() - timedelta(days=self._lag_days) + common_tags = [ + "source:anthropic_analytics", + "org_id:{}".format(self._org_id), + ] + + client = AnthropicAnalyticsClient(self.http, self._api_key, self.log) + + try: + # /summaries -- single row (or None if not yet available) + summary_row = client.summaries(report_date) + self._emit(mappers.from_summaries(summary_row, report_date), common_tags) + + # /users -- paginated + self._emit(mappers.from_users(client.users(report_date), report_date), common_tags) + + # /usage_report, /cost_report -- grouped by model+product + self._emit( + mappers.from_usage_report(client.usage_report(report_date), report_date), + common_tags, + ) + self._emit( + mappers.from_cost_report(client.cost_report(report_date), report_date), + common_tags, + ) + + # /user_usage_report, /user_cost_report -- per-user + self._emit( + mappers.from_user_usage_report(client.user_usage_report(report_date), report_date), + common_tags, + ) + self._emit( + mappers.from_user_cost_report(client.user_cost_report(report_date), report_date), + common_tags, + ) + except Exception as e: + self.log.exception("Failed to collect Claude Enterprise Analytics for %s", report_date) + self.service_check( + _SERVICE_CHECK_API, + AgentCheck.CRITICAL, + tags=common_tags, + message=str(e)[:200], + ) + raise + + self.service_check(_SERVICE_CHECK_API, AgentCheck.OK, tags=common_tags) + + def _emit(self, triples, common_tags): + instance_tags = list(self.instance.get("tags") or []) + for name, value, extra_tags in triples: + self.gauge(name, value, tags=common_tags + instance_tags + list(extra_tags)) diff --git a/claude_enterprise_analytics/datadog_checks/claude_enterprise_analytics/config_models/__init__.py b/claude_enterprise_analytics/datadog_checks/claude_enterprise_analytics/config_models/__init__.py new file mode 100644 index 0000000000..5c2bf5c9f4 --- /dev/null +++ b/claude_enterprise_analytics/datadog_checks/claude_enterprise_analytics/config_models/__init__.py @@ -0,0 +1,20 @@ +# This file is autogenerated. +# To change this file you should edit assets/configuration/spec.yaml and then run the following commands: +# ddev -x validate config -s +# ddev -x validate models -s + +from .instance import InstanceConfig +from .shared import SharedConfig + + +class ConfigMixin: + _config_model_instance: InstanceConfig + _config_model_shared: SharedConfig + + @property + def config(self) -> InstanceConfig: + return self._config_model_instance + + @property + def shared_config(self) -> SharedConfig: + return self._config_model_shared diff --git a/claude_enterprise_analytics/datadog_checks/claude_enterprise_analytics/config_models/defaults.py b/claude_enterprise_analytics/datadog_checks/claude_enterprise_analytics/config_models/defaults.py new file mode 100644 index 0000000000..b7932b83b5 --- /dev/null +++ b/claude_enterprise_analytics/datadog_checks/claude_enterprise_analytics/config_models/defaults.py @@ -0,0 +1,28 @@ +# This file is autogenerated. +# To change this file you should edit assets/configuration/spec.yaml and then run the following commands: +# ddev -x validate config -s +# ddev -x validate models -s + + +def instance_disable_generic_tags(): + return False + + +def instance_empty_default_hostname(): + return False + + +def instance_enable_legacy_tags_normalization(): + return True + + +def instance_lag_days(): + return 3 + + +def instance_min_collection_interval(): + return 15 + + +def instance_org_id(): + return 'my-org' diff --git a/claude_enterprise_analytics/datadog_checks/claude_enterprise_analytics/config_models/instance.py b/claude_enterprise_analytics/datadog_checks/claude_enterprise_analytics/config_models/instance.py new file mode 100644 index 0000000000..28ebe546bc --- /dev/null +++ b/claude_enterprise_analytics/datadog_checks/claude_enterprise_analytics/config_models/instance.py @@ -0,0 +1,61 @@ +# This file is autogenerated. +# To change this file you should edit assets/configuration/spec.yaml and then run the following commands: +# ddev -x validate config -s +# ddev -x validate models -s + +from __future__ import annotations + +from typing import Optional + +from pydantic import BaseModel, ConfigDict, field_validator, model_validator + +from datadog_checks.base.utils.functions import identity +from datadog_checks.base.utils.models import validation + +from . import defaults, validators + + +class MetricPatterns(BaseModel): + model_config = ConfigDict( + arbitrary_types_allowed=True, + frozen=True, + ) + exclude: Optional[tuple[str, ...]] = None + include: Optional[tuple[str, ...]] = None + + +class InstanceConfig(BaseModel): + model_config = ConfigDict( + validate_default=True, + arbitrary_types_allowed=True, + frozen=True, + ) + anthropic_api_key: str + disable_generic_tags: Optional[bool] = None + empty_default_hostname: Optional[bool] = None + enable_legacy_tags_normalization: Optional[bool] = None + lag_days: Optional[int] = None + metric_patterns: Optional[MetricPatterns] = None + min_collection_interval: Optional[float] = None + org_id: Optional[str] = None + service: Optional[str] = None + tags: Optional[tuple[str, ...]] = None + + @model_validator(mode='before') + def _initial_validation(cls, values): + return validation.core.initialize_config(getattr(validators, 'initialize_instance', identity)(values)) + + @field_validator('*', mode='before') + def _validate(cls, value, info): + field = cls.model_fields[info.field_name] + field_name = field.alias or info.field_name + if field_name in info.context['configured_fields']: + value = getattr(validators, f'instance_{info.field_name}', identity)(value, field=field) + else: + value = getattr(defaults, f'instance_{info.field_name}', lambda: value)() + + return validation.utils.make_immutable(value) + + @model_validator(mode='after') + def _final_validation(cls, model): + return validation.core.check_model(getattr(validators, 'check_instance', identity)(model)) diff --git a/claude_enterprise_analytics/datadog_checks/claude_enterprise_analytics/config_models/shared.py b/claude_enterprise_analytics/datadog_checks/claude_enterprise_analytics/config_models/shared.py new file mode 100644 index 0000000000..4017c43e2e --- /dev/null +++ b/claude_enterprise_analytics/datadog_checks/claude_enterprise_analytics/config_models/shared.py @@ -0,0 +1,41 @@ +# This file is autogenerated. +# To change this file you should edit assets/configuration/spec.yaml and then run the following commands: +# ddev -x validate config -s +# ddev -x validate models -s + +from __future__ import annotations + +from typing import Optional + +from pydantic import BaseModel, ConfigDict, field_validator, model_validator + +from datadog_checks.base.utils.functions import identity +from datadog_checks.base.utils.models import validation + +from . import validators + + +class SharedConfig(BaseModel): + model_config = ConfigDict( + validate_default=True, + arbitrary_types_allowed=True, + frozen=True, + ) + service: Optional[str] = None + + @model_validator(mode='before') + def _initial_validation(cls, values): + return validation.core.initialize_config(getattr(validators, 'initialize_shared', identity)(values)) + + @field_validator('*', mode='before') + def _validate(cls, value, info): + field = cls.model_fields[info.field_name] + field_name = field.alias or info.field_name + if field_name in info.context['configured_fields']: + value = getattr(validators, f'shared_{info.field_name}', identity)(value, field=field) + + return validation.utils.make_immutable(value) + + @model_validator(mode='after') + def _final_validation(cls, model): + return validation.core.check_model(getattr(validators, 'check_shared', identity)(model)) diff --git a/claude_enterprise_analytics/datadog_checks/claude_enterprise_analytics/config_models/validators.py b/claude_enterprise_analytics/datadog_checks/claude_enterprise_analytics/config_models/validators.py new file mode 100644 index 0000000000..5e48f02a73 --- /dev/null +++ b/claude_enterprise_analytics/datadog_checks/claude_enterprise_analytics/config_models/validators.py @@ -0,0 +1,13 @@ +# (C) Datadog, Inc. 2026-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) + +# Here you can include additional config validators or transformers +# +# def initialize_instance(values, **kwargs): +# if 'my_option' not in values and 'my_legacy_option' in values: +# values['my_option'] = values['my_legacy_option'] +# if values.get('my_number') > 10: +# raise ValueError('my_number max value is 10, got %s' % str(values.get('my_number'))) +# +# return values diff --git a/claude_enterprise_analytics/datadog_checks/claude_enterprise_analytics/data/conf.yaml.example b/claude_enterprise_analytics/datadog_checks/claude_enterprise_analytics/data/conf.yaml.example new file mode 100644 index 0000000000..edcf6c8315 --- /dev/null +++ b/claude_enterprise_analytics/datadog_checks/claude_enterprise_analytics/data/conf.yaml.example @@ -0,0 +1,75 @@ +## All options defined here are available to all instances. +# +init_config: + + ## @param service - string - optional + ## Attach the tag `service:` to every metric, event, and service check emitted by this integration. + ## + ## Additionally, this sets the default `service` for every log source. + # + # service: + +## Every instance is scheduled independently of the others. +# +instances: + + ## @param anthropic_api_key - string - required + ## API key for the Anthropic Claude Enterprise Analytics API. Generate one at + ## https://claude.ai/analytics/api-keys (requires Primary Owner role). Must + ## carry the `read:analytics` scope. + # + - anthropic_api_key: + + ## @param org_id - string - optional - default: my-org + ## Free-form organization identifier emitted as the `org_id` tag on every + ## metric. Useful when more than one Anthropic org reports into the same + ## Datadog account. + # + # org_id: my-org + + ## @param lag_days - integer - optional - default: 3 + ## How many days behind today to poll. Anthropic publishes daily aggregates + ## with a roughly 3-day lag; the default reflects that. Increase if you see + ## empty responses for the default date. + # + # lag_days: 3 + + ## @param tags - list of strings - optional + ## A list of tags to attach to every metric and service check emitted by this instance. + ## + ## Learn more about tagging at https://docs.datadoghq.com/tagging + # + # tags: + # - : + # - : + + ## @param service - string - optional + ## Attach the tag `service:` to every metric, event, and service check emitted by this integration. + ## + ## Overrides any `service` defined in the `init_config` section. + # + # service: + + ## @param min_collection_interval - number - optional - default: 15 + ## This changes the collection interval of the check. For more information, see: + ## https://docs.datadoghq.com/developers/write_agent_check/#collection-interval + # + # min_collection_interval: 15 + + ## @param empty_default_hostname - boolean - optional - default: false + ## This forces the check to send metrics with no hostname. + ## + ## This is useful for cluster-level checks. + # + # empty_default_hostname: false + + ## @param metric_patterns - mapping - optional + ## A mapping of metrics to include or exclude, with each entry being a regular expression. + ## + ## Metrics defined in `exclude` will take precedence in case of overlap. + # + # metric_patterns: + # include: + # - + # exclude: + # - diff --git a/claude_enterprise_analytics/hatch.toml b/claude_enterprise_analytics/hatch.toml new file mode 100644 index 0000000000..fee2455000 --- /dev/null +++ b/claude_enterprise_analytics/hatch.toml @@ -0,0 +1,4 @@ +[env.collectors.datadog-checks] + +[[envs.default.matrix]] +python = ["3.13"] diff --git a/claude_enterprise_analytics/images/IMAGES_README.md b/claude_enterprise_analytics/images/IMAGES_README.md new file mode 100644 index 0000000000..443f3c45e3 --- /dev/null +++ b/claude_enterprise_analytics/images/IMAGES_README.md @@ -0,0 +1,41 @@ +# Marketplace Media Carousel Guidelines + +## Using the media gallery + +Please upload images to use the media gallery. Integrations require a minimum of 3 images. Images should highlight your product, your integration, and a full image of the Datadog integration dashboard. The gallery +can hold a maximum of 8 pieces of media total, and one of these pieces of media +can be a video (guidelines and submission steps below). Images should be +added to your /images directory and referenced in the manifest.json file. + + +## Image and video requirements + +### Images + +``` +File type : .jpg or .png +File size : ~500 KB per image, with a max of 1 MB per image +File dimensions : The image must be between 1440px and 2880px width, with a 16:9 aspect ratio (for example: 1440x810) +File name : Use only letters, numbers, underscores, and hyphens +Color mode : RGB +Color profile : sRGB +Description : 300 characters maximum +``` + +### Video + +To display a video in your media gallery, please send our team the zipped file +or a link to download the video at `marketplace@datadog.com`. In addition, +please upload a thumbnail image for your video as a part of the pull request. +Once approved, we will upload the file to Vimeo and provide you with the +vimeo_id to add to your manifest.json file. Please note that the gallery can +only hold one video. + +``` +File type : MP4 H.264 +File size : Max 1 video; 1 GB maximum size +File dimensions : The aspect ratio must be exactly 16:9, and the resolution must be 1920x1080 or higher +File name : partnerName-appName.mp4 +Run time : Recommendation of 60 seconds or less +Description : 300 characters maximum +``` diff --git a/claude_enterprise_analytics/manifest.json b/claude_enterprise_analytics/manifest.json new file mode 100644 index 0000000000..ebdf250fa5 --- /dev/null +++ b/claude_enterprise_analytics/manifest.json @@ -0,0 +1,58 @@ +{ + "manifest_version": "2.0.0", + "app_uuid": "543252ff-5063-4617-bc6e-230703cebad6", + "app_id": "claude-enterprise-analytics", + "display_on_public_website": false, + "owner": "integrations-developer-platform", + "tile": { + "overview": "README.md#Overview", + "configuration": "README.md#Setup", + "support": "README.md#Support", + "changelog": "CHANGELOG.md", + "description": "Unofficial integration: usage, cost, and seat-utilization metrics from the Claude Enterprise Analytics API.", + "title": "Claude Enterprise Analytics", + "media": [], + "classifier_tags": [ + "Supported OS::Linux", + "Supported OS::Windows", + "Supported OS::macOS", + "Category::AI/ML", + "Category::Cost Management", + "Category::Metrics", + "Offering::Integration", + "Submitted Data Type::Metrics" + ] + }, + "assets": { + "integration": { + "auto_install": true, + "source_type_id": 79445034, + "source_type_name": "Claude Enterprise Analytics", + "configuration": { + "spec": "assets/configuration/spec.yaml" + }, + "events": { + "creates_events": false + }, + "metrics": { + "prefix": "claude_enterprise_analytics.", + "check": "claude_enterprise_analytics.org.seats_assigned", + "metadata_path": "metadata.csv" + }, + "service_checks": { + "metadata_path": "assets/service_checks.json" + } + }, + "dashboards": { + "Claude Enterprise Analytics Overview": "assets/dashboards/claude_enterprise_analytics_overview.json" + }, + "monitors": {}, + "saved_views": {} + }, + "author": { + "support_email": "ops@typeform.com", + "name": "Typeform Platform", + "homepage": "https://www.typeform.com", + "sales_email": "ops@typeform.com" + } +} diff --git a/claude_enterprise_analytics/metadata.csv b/claude_enterprise_analytics/metadata.csv new file mode 100644 index 0000000000..202015df14 --- /dev/null +++ b/claude_enterprise_analytics/metadata.csv @@ -0,0 +1,46 @@ +metric_name,metric_type,interval,unit_name,per_unit_name,description,orientation,integration,short_name,curated_metric,sample_tags +claude_enterprise_analytics.org.dau,gauge,,,,Daily active users in the Claude org for the report_date.,0,claude_enterprise_analytics,DAU,,"report_date,org_id" +claude_enterprise_analytics.org.wau,gauge,,,,Weekly active users in the Claude org for the report_date.,0,claude_enterprise_analytics,WAU,,"report_date,org_id" +claude_enterprise_analytics.org.mau,gauge,,,,Monthly active users in the Claude org for the report_date.,0,claude_enterprise_analytics,MAU,,"report_date,org_id" +claude_enterprise_analytics.org.seats_assigned,gauge,,,,Total Claude seats assigned in the org as of the report_date.,0,claude_enterprise_analytics,seats assigned,,"report_date,org_id" +claude_enterprise_analytics.org.invites_pending,gauge,,,,Pending Claude seat invitations as of the report_date.,0,claude_enterprise_analytics,pending invites,,"report_date,org_id" +claude_enterprise_analytics.org.adoption_rate.daily,gauge,,percent,,Daily adoption rate: DAU / assigned_seats as reported by the API.,0,claude_enterprise_analytics,daily adoption,,"report_date,org_id" +claude_enterprise_analytics.org.adoption_rate.weekly,gauge,,percent,,Weekly adoption rate: WAU / assigned_seats as reported by the API.,0,claude_enterprise_analytics,weekly adoption,,"report_date,org_id" +claude_enterprise_analytics.org.adoption_rate.monthly,gauge,,percent,,Monthly adoption rate: MAU / assigned_seats as reported by the API.,0,claude_enterprise_analytics,monthly adoption,,"report_date,org_id" +claude_enterprise_analytics.org.cowork.dau,gauge,,,,Daily active users of the Claude Cowork product for the report_date.,0,claude_enterprise_analytics,cowork DAU,,"report_date,org_id" +claude_enterprise_analytics.org.cowork.wau,gauge,,,,Weekly active users of the Claude Cowork product for the report_date.,0,claude_enterprise_analytics,cowork WAU,,"report_date,org_id" +claude_enterprise_analytics.org.cowork.mau,gauge,,,,Monthly active users of the Claude Cowork product for the report_date.,0,claude_enterprise_analytics,cowork MAU,,"report_date,org_id" +claude_enterprise_analytics.user.chat.messages,gauge,,,,Per-user chat message count for the report_date.,0,claude_enterprise_analytics,chat messages,,"report_date,user_email,org_id" +claude_enterprise_analytics.user.chat.conversations,gauge,,,,Per-user distinct chat conversation count for the report_date.,0,claude_enterprise_analytics,chat conversations,,"report_date,user_email,org_id" +claude_enterprise_analytics.user.chat.thinking_messages,gauge,,,,Per-user chat thinking-mode message count for the report_date.,0,claude_enterprise_analytics,thinking messages,,"report_date,user_email,org_id" +claude_enterprise_analytics.user.projects.used,gauge,,,,Per-user distinct projects used for the report_date.,0,claude_enterprise_analytics,projects used,,"report_date,user_email,org_id" +claude_enterprise_analytics.user.projects.created,gauge,,,,Per-user distinct projects created for the report_date.,0,claude_enterprise_analytics,projects created,,"report_date,user_email,org_id" +claude_enterprise_analytics.user.artifacts.created,gauge,,,,Per-user distinct artifacts created for the report_date.,0,claude_enterprise_analytics,artifacts created,,"report_date,user_email,org_id" +claude_enterprise_analytics.user.skills.used,gauge,,,,Per-user distinct skills used for the report_date.,0,claude_enterprise_analytics,skills used,,"report_date,user_email,org_id" +claude_enterprise_analytics.user.connectors.used,gauge,,,,Per-user MCP/connector usage count for the report_date.,0,claude_enterprise_analytics,connectors used,,"report_date,user_email,org_id" +claude_enterprise_analytics.user.files.uploaded,gauge,,,,Per-user distinct files uploaded to chat for the report_date.,0,claude_enterprise_analytics,files uploaded,,"report_date,user_email,org_id" +claude_enterprise_analytics.user.claude_code.sessions,gauge,,,,Per-user Claude Code session count for the report_date.,0,claude_enterprise_analytics,Claude Code sessions,,"report_date,user_email,org_id" +claude_enterprise_analytics.user.claude_code.commits,gauge,,,,Per-user Claude Code commit count for the report_date.,0,claude_enterprise_analytics,Claude Code commits,,"report_date,user_email,org_id" +claude_enterprise_analytics.user.claude_code.prs,gauge,,,,Per-user Claude Code pull-request count for the report_date.,0,claude_enterprise_analytics,Claude Code PRs,,"report_date,user_email,org_id" +claude_enterprise_analytics.user.claude_code.lines_added,gauge,,,,Per-user Claude Code lines-of-code added for the report_date.,0,claude_enterprise_analytics,lines added,,"report_date,user_email,org_id" +claude_enterprise_analytics.user.claude_code.lines_removed,gauge,,,,Per-user Claude Code lines-of-code removed for the report_date.,0,claude_enterprise_analytics,lines removed,,"report_date,user_email,org_id" +claude_enterprise_analytics.user.claude_code.tool_actions,gauge,,,,Per-user Claude Code tool action count for the report_date; tags `tool` and `outcome` (accepted/rejected) split this.,0,claude_enterprise_analytics,tool actions,,"report_date,user_email,tool,outcome,org_id" +claude_enterprise_analytics.user.web_search_count,gauge,,,,Per-user web-search invocation count for the report_date.,0,claude_enterprise_analytics,user web searches,,"report_date,user_email,org_id" +claude_enterprise_analytics.tokens.uncached_input,gauge,,,,Org-wide uncached input tokens consumed for the report_date.,0,claude_enterprise_analytics,uncached input tokens,,"report_date,model,product,context_window,org_id" +claude_enterprise_analytics.tokens.cache_read,gauge,,,,Org-wide cache-read input tokens consumed for the report_date.,0,claude_enterprise_analytics,cache read tokens,,"report_date,model,product,context_window,org_id" +claude_enterprise_analytics.tokens.cache_write_1h,gauge,,,,Org-wide 1-hour ephemeral cache-write tokens for the report_date.,0,claude_enterprise_analytics,cache write 1h,,"report_date,model,product,context_window,org_id" +claude_enterprise_analytics.tokens.cache_write_5m,gauge,,,,Org-wide 5-minute ephemeral cache-write tokens for the report_date.,0,claude_enterprise_analytics,cache write 5m,,"report_date,model,product,context_window,org_id" +claude_enterprise_analytics.tokens.output,gauge,,,,Org-wide output tokens for the report_date.,0,claude_enterprise_analytics,output tokens,,"report_date,model,product,context_window,org_id" +claude_enterprise_analytics.requests,gauge,,request,,Org-wide API request count for the report_date.,0,claude_enterprise_analytics,requests,,"report_date,model,product,context_window,org_id" +claude_enterprise_analytics.web_search_requests,gauge,,request,,Org-wide web-search tool invocations for the report_date.,0,claude_enterprise_analytics,web search requests,,"report_date,model,product,context_window,org_id" +claude_enterprise_analytics.cost.amount_usd,gauge,,dollar,,Org-wide cost (USD) for the report_date; the API only exposes list pricing.,0,claude_enterprise_analytics,cost USD,,"report_date,model,product,currency,org_id" +claude_enterprise_analytics.cost.list_amount_usd,gauge,,dollar,,Org-wide list-price cost (USD) for the report_date.,0,claude_enterprise_analytics,list cost USD,,"report_date,model,product,currency,org_id" +claude_enterprise_analytics.user.tokens_total,gauge,,,,Per-user total tokens (input + output + cache) for the report_date.,0,claude_enterprise_analytics,total tokens,,"report_date,user_email,org_id" +claude_enterprise_analytics.user.uncached_input_tokens,gauge,,,,Per-user uncached input tokens for the report_date.,0,claude_enterprise_analytics,uncached input,,"report_date,user_email,org_id" +claude_enterprise_analytics.user.output_tokens,gauge,,,,Per-user output tokens for the report_date.,0,claude_enterprise_analytics,output tokens,,"report_date,user_email,org_id" +claude_enterprise_analytics.user.cache_read_tokens,gauge,,,,Per-user cache-read input tokens for the report_date.,0,claude_enterprise_analytics,cache read,,"report_date,user_email,org_id" +claude_enterprise_analytics.user.cache_write_5m_tokens,gauge,,,,Per-user 5-minute ephemeral cache-write tokens for the report_date.,0,claude_enterprise_analytics,cache write 5m,,"report_date,user_email,org_id" +claude_enterprise_analytics.user.cache_write_1h_tokens,gauge,,,,Per-user 1-hour ephemeral cache-write tokens for the report_date.,0,claude_enterprise_analytics,cache write 1h,,"report_date,user_email,org_id" +claude_enterprise_analytics.user.requests,gauge,,request,,Per-user API request count for the report_date.,0,claude_enterprise_analytics,user requests,,"report_date,user_email,org_id" +claude_enterprise_analytics.user.cost.amount_usd,gauge,,dollar,,Per-user cost (USD) for the report_date; the API only exposes list pricing.,0,claude_enterprise_analytics,user cost USD,,"report_date,user_email,currency,org_id" +claude_enterprise_analytics.user.cost.list_amount_usd,gauge,,dollar,,Per-user list-price cost (USD) for the report_date.,0,claude_enterprise_analytics,user list cost USD,,"report_date,user_email,currency,org_id" diff --git a/claude_enterprise_analytics/pyproject.toml b/claude_enterprise_analytics/pyproject.toml new file mode 100644 index 0000000000..68f9bc1de6 --- /dev/null +++ b/claude_enterprise_analytics/pyproject.toml @@ -0,0 +1,60 @@ +[build-system] +requires = [ + "hatchling>=0.13.0", +] +build-backend = "hatchling.build" + +[project] +name = "datadog-claude-enterprise-analytics" +description = "The Claude Enterprise Analytics check" +readme = "README.md" +license = "BSD-3-Clause" +requires-python = ">=3.13" +keywords = [ + "datadog", + "datadog agent", + "datadog check", + "claude_enterprise_analytics", +] +authors = [ + { name = "Typeform Platform", email = "ops@typeform.com" }, +] +classifiers = [ + "Development Status :: 5 - Production/Stable", + "Intended Audience :: Developers", + "Intended Audience :: System Administrators", + "License :: OSI Approved :: BSD License", + "Private :: Do Not Upload", + "Programming Language :: Python :: 3.13", + "Topic :: System :: Monitoring", +] +dependencies = [ + "datadog-checks-base>=37.33.0", +] +dynamic = [ + "version", +] + +[project.optional-dependencies] +deps = [] + +[project.urls] +Source = "https://github.com/DataDog/integrations-extras" + +[tool.hatch.version] +path = "datadog_checks/claude_enterprise_analytics/__about__.py" + +[tool.hatch.build.targets.sdist] +include = [ + "/datadog_checks", + "/tests", + "/manifest.json", +] + +[tool.hatch.build.targets.wheel] +include = [ + "/datadog_checks/claude_enterprise_analytics", +] +dev-mode-dirs = [ + ".", +] diff --git a/claude_enterprise_analytics/tests/__init__.py b/claude_enterprise_analytics/tests/__init__.py new file mode 100644 index 0000000000..75c6647cb9 --- /dev/null +++ b/claude_enterprise_analytics/tests/__init__.py @@ -0,0 +1,3 @@ +# (C) Datadog, Inc. 2026-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) diff --git a/claude_enterprise_analytics/tests/conftest.py b/claude_enterprise_analytics/tests/conftest.py new file mode 100644 index 0000000000..284a75e390 --- /dev/null +++ b/claude_enterprise_analytics/tests/conftest.py @@ -0,0 +1,18 @@ +# (C) Typeform Platform 2026-present +# Unofficial integration. Not affiliated with Anthropic, PBC. +# Licensed under a 3-clause BSD style license (see LICENSE) +import pytest + + +@pytest.fixture(scope='session') +def dd_environment(): + yield + + +@pytest.fixture +def instance(): + return { + "anthropic_api_key": "test-key-not-real", + "org_id": "test-org", + "lag_days": 3, + } diff --git a/claude_enterprise_analytics/tests/test_unit.py b/claude_enterprise_analytics/tests/test_unit.py new file mode 100644 index 0000000000..f959e76534 --- /dev/null +++ b/claude_enterprise_analytics/tests/test_unit.py @@ -0,0 +1,413 @@ +# (C) Typeform Platform 2026-present +# Unofficial integration. Not affiliated with Anthropic, PBC. +# Licensed under a 3-clause BSD style license (see LICENSE) + +from datetime import date +from typing import Any, Callable, Dict # noqa: F401 +from unittest import mock + +import pytest + +from datadog_checks.base import AgentCheck, ConfigurationError # noqa: F401 +from datadog_checks.base.stubs.aggregator import AggregatorStub # noqa: F401 +from datadog_checks.claude_enterprise_analytics import ClaudeEnterpriseAnalyticsCheck +from datadog_checks.claude_enterprise_analytics import _mappers as mappers +from datadog_checks.claude_enterprise_analytics._anthropic_client import AnthropicAnalyticsClient + +REPORT_DATE = date(2026, 5, 18) + + +# ---------- mapper helper unit tests --------------------------------------- + + +def test_num_handles_string_none_and_invalid(): + assert mappers._num("1.5") == 1.5 + assert mappers._num(2) == 2.0 + assert mappers._num(None) == 0.0 + assert mappers._num("") == 0.0 + assert mappers._num("not-a-number") == 0.0 + + +def test_cents_to_usd_divides_by_100(): + assert mappers._cents_to_usd("131309.570280") == pytest.approx(1313.0957028) + assert mappers._cents_to_usd(None) == 0.0 + assert mappers._cents_to_usd("") == 0.0 + + +def test_g_walks_nested_dicts_with_default(): + obj = {"a": {"b": {"c": 42}}} + assert mappers._g(obj, "a", "b", "c") == 42 + assert mappers._g(obj, "a", "missing", default="x") == "x" + assert mappers._g(None, "a") == 0 + assert mappers._g({"a": None}, "a", "b", default=7) == 7 + + +def test_tag_normalizes_missing_values(): + assert mappers._tag("model", "claude-opus") == "model:claude-opus" + assert mappers._tag("model", None) == "model:unknown" + assert mappers._tag("model", "") == "model:unknown" + + +# ---------- mapper function tests ------------------------------------------ + + +def test_summaries_emits_expected_metrics(): + row = { + "daily_active_user_count": 95, + "weekly_active_user_count": 135, + "monthly_active_user_count": 143, + "assigned_seat_count": 150, + "pending_invite_count": 0, + "daily_adoption_rate": 63.33, + "weekly_adoption_rate": 90.0, + "monthly_adoption_rate": 95.33, + } + out = list(mappers.from_summaries(row, REPORT_DATE)) + names = {m for m, _, _ in out} + assert "org.dau" in names + assert "org.seats_assigned" in names + assert "org.adoption_rate.daily" in names + for _, _, tags in out: + assert "report_date:2026-05-18" in tags + + +def test_summaries_with_none_row_emits_nothing(): + assert list(mappers.from_summaries(None, REPORT_DATE)) == [] + + +def test_users_emits_chat_claude_code_and_tool_action_metrics(): + rows = [ + { + "user": {"email_address": "alice@example.com"}, + "chat_metrics": { + "message_count": 12, + "distinct_conversation_count": 3, + "thinking_message_count": 1, + "distinct_projects_used_count": 2, + "distinct_projects_created_count": 1, + "distinct_artifacts_created_count": 4, + "distinct_skills_used_count": 0, + "connectors_used_count": 0, + "distinct_files_uploaded_count": 5, + }, + "claude_code_metrics": { + "core_metrics": { + "distinct_session_count": 22, + "commit_count": 3, + "pull_request_count": 1, + "lines_of_code": {"added_count": 1757, "removed_count": 669}, + }, + "tool_actions": { + "edit_tool": {"accepted_count": 90, "rejected_count": 2}, + "write_tool": {"accepted_count": 11, "rejected_count": 0}, + "broken": "ignored", + }, + }, + "web_search_count": 7, + } + ] + out = list(mappers.from_users(rows, REPORT_DATE)) + by_name_tags = {(m, tuple(sorted(t))): v for m, v, t in out} + + # Chat surface + assert any(m == "user.chat.messages" and v == 12 for m, v, _ in out) + assert any(m == "user.artifacts.created" and v == 4 for m, v, _ in out) + assert any(m == "user.files.uploaded" and v == 5 for m, v, _ in out) + + # Claude Code core + assert any(m == "user.claude_code.sessions" and v == 22 for m, v, _ in out) + assert any(m == "user.claude_code.commits" and v == 3 for m, v, _ in out) + assert any(m == "user.claude_code.lines_added" and v == 1757 for m, v, _ in out) + assert any(m == "user.claude_code.lines_removed" and v == 669 for m, v, _ in out) + + # Tool actions are tagged by tool + outcome; `broken` (non-dict) is skipped. + tool_metrics = [(m, v, t) for m, v, t in out if m == "user.claude_code.tool_actions"] + assert len(tool_metrics) == 4 # 2 tools x 2 outcomes + assert ( + "user.claude_code.tool_actions", + 90.0, + ["report_date:2026-05-18", "user_email:alice@example.com", "tool:edit", "outcome:accepted"], + ) in [(m, v, t) for m, v, t in tool_metrics] + + # Web search + assert any(m == "user.web_search_count" and v == 7 for m, v, _ in out) + + # Every row carries user_email tag + for _, _, tags in out: + assert "user_email:alice@example.com" in tags + + # silence unused-var warning + _ = by_name_tags + + +def test_users_handles_missing_optional_subdocs(): + rows = [{"user": {"email_address": "bob@example.com"}}] + out = list(mappers.from_users(rows, REPORT_DATE)) + # Should still emit zero-valued metrics, not crash. + assert any(m == "user.chat.messages" and v == 0 for m, v, _ in out) + # No tool_actions yielded since the dict is missing. + assert not any(m == "user.claude_code.tool_actions" for m, _, _ in out) + + +def test_usage_report_emits_seven_metrics_per_row(): + rows = [ + { + "model": "claude-opus-4-7", + "product": "claude_code", + "context_window": None, + "uncached_input_tokens": 10243631, + "output_tokens": 7705210, + "cache_read_input_tokens": 1746957745, + "cache_creation": { + "ephemeral_1h_input_tokens": 52015263, + "ephemeral_5m_input_tokens": 21611713, + }, + "server_tool_use": {"web_search_requests": 19}, + "requests": 21833, + } + ] + out = list(mappers.from_usage_report(rows, REPORT_DATE)) + by_name = {m: v for m, v, _ in out} + assert by_name["tokens.uncached_input"] == 10243631 + assert by_name["tokens.output"] == 7705210 + assert by_name["tokens.cache_read"] == 1746957745 + assert by_name["tokens.cache_write_1h"] == 52015263 + assert by_name["tokens.cache_write_5m"] == 21611713 + assert by_name["requests"] == 21833 + assert by_name["web_search_requests"] == 19 + # Tags include model + product + context_window + for _, _, tags in out: + assert "model:claude-opus-4-7" in tags + assert "product:claude_code" in tags + assert "context_window:unknown" in tags # None -> 'unknown' + + +def test_cost_report_divides_amount_by_100(): + rows = [ + { + "model": "claude-opus-4-7", + "product": "claude_code", + "currency": "USD", + "amount": "8164942.9775", + "list_amount": "8164942.9775", + } + ] + out = list(mappers.from_cost_report(rows, REPORT_DATE)) + by_name = {name: value for name, value, _ in out} + assert by_name["cost.amount_usd"] == pytest.approx(81649.429775) + assert by_name["cost.list_amount_usd"] == pytest.approx(81649.429775) + + +def test_user_usage_report_emits_all_per_user_token_metrics(): + rows = [ + { + "actor": {"email": "alice@example.com"}, + "total_tokens": 271849312, + "uncached_input_tokens": 146848, + "output_tokens": 447409, + "cache_read_input_tokens": 268750070, + "cache_creation": { + "ephemeral_1h_input_tokens": 2192245, + "ephemeral_5m_input_tokens": 312740, + }, + "requests": 853, + } + ] + out = list(mappers.from_user_usage_report(rows, REPORT_DATE)) + by_name = {m: v for m, v, _ in out} + assert by_name["user.tokens_total"] == 271849312 + assert by_name["user.uncached_input_tokens"] == 146848 + assert by_name["user.output_tokens"] == 447409 + assert by_name["user.cache_read_tokens"] == 268750070 + assert by_name["user.cache_write_1h_tokens"] == 2192245 + assert by_name["user.cache_write_5m_tokens"] == 312740 + assert by_name["user.requests"] == 853 + # 7 metrics per user + assert len(out) == 7 + + +def test_user_cost_report_divides_per_user_amount_by_100(): + rows = [ + { + "actor": {"email": "alice@example.com"}, + "currency": "USD", + "amount": "1683895.6900", + "list_amount": "1683895.6900", + } + ] + out = list(mappers.from_user_cost_report(rows, REPORT_DATE)) + by_name = {m: v for m, v, _ in out} + assert by_name["user.cost.amount_usd"] == pytest.approx(16838.956900) + assert by_name["user.cost.list_amount_usd"] == pytest.approx(16838.956900) + + +# ---------- Anthropic client tests ----------------------------------------- + + +def _mock_response(json_payload, status_code=200, headers=None): + resp = mock.MagicMock() + resp.status_code = status_code + resp.json.return_value = json_payload + resp.headers = headers or {} + resp.text = "" + resp.raise_for_status = mock.MagicMock() + if status_code >= 400: + resp.raise_for_status.side_effect = Exception("HTTP {}".format(status_code)) + return resp + + +def test_client_summaries_calls_correct_url_and_returns_first_row(): + http = mock.MagicMock() + http.get.return_value = _mock_response({"summaries": [{"daily_active_user_count": 95}]}) + client = AnthropicAnalyticsClient(http, "secret-key", mock.MagicMock()) + + row = client.summaries(REPORT_DATE) + + assert row == {"daily_active_user_count": 95} + call = http.get.call_args + assert call.args[0].endswith("/summaries") + # strict-before range: [day, day+1) + assert call.kwargs["params"]["starting_date"] == "2026-05-18" + assert call.kwargs["params"]["ending_date"] == "2026-05-19" + assert call.kwargs["headers"]["x-api-key"] == "secret-key" + + +def test_client_summaries_returns_none_when_empty(): + http = mock.MagicMock() + http.get.return_value = _mock_response({"summaries": []}) + client = AnthropicAnalyticsClient(http, "k", mock.MagicMock()) + assert client.summaries(REPORT_DATE) is None + + +def test_client_paginates_via_next_page_token(): + http = mock.MagicMock() + http.get.side_effect = [ + _mock_response({"data": [{"id": 1}, {"id": 2}], "next_page": "page_abc", "has_more": True}), + _mock_response({"data": [{"id": 3}], "next_page": None, "has_more": False}), + ] + client = AnthropicAnalyticsClient(http, "k", mock.MagicMock()) + + rows = list(client.users(REPORT_DATE)) + + assert [r["id"] for r in rows] == [1, 2, 3] + assert http.get.call_count == 2 + second_call_params = http.get.call_args_list[1].kwargs["params"] + assert second_call_params["page"] == "page_abc" + + +def test_client_iter_report_groups_by_model_and_product(): + http = mock.MagicMock() + http.get.return_value = _mock_response( + {"data": [{"results": [{"model": "claude-opus-4-7", "amount": "100"}]}], "has_more": False, "next_page": None} + ) + client = AnthropicAnalyticsClient(http, "k", mock.MagicMock()) + + rows = list(client.cost_report(REPORT_DATE)) + + assert rows == [{"model": "claude-opus-4-7", "amount": "100"}] + params = http.get.call_args.kwargs["params"] + assert params["group_by[]"] == ["model", "product"] + assert params["starting_at"] == "2026-05-18T00:00:00Z" + assert params["ending_at"] == "2026-05-19T00:00:00Z" + + +def test_client_retries_on_429_then_succeeds(): + http = mock.MagicMock() + http.get.side_effect = [ + _mock_response({}, status_code=429, headers={"retry-after": "0"}), + _mock_response({"summaries": [{"daily_active_user_count": 1}]}), + ] + client = AnthropicAnalyticsClient(http, "k", mock.MagicMock()) + + with mock.patch("time.sleep") as msleep: + row = client.summaries(REPORT_DATE) + + assert row == {"daily_active_user_count": 1} + assert http.get.call_count == 2 + assert msleep.called + + +# ---------- check-level integration tests ---------------------------------- + + +def _empty_payload(rows_key="data"): + return {rows_key: [], "next_page": None, "has_more": False} + + +def test_check_emits_metrics_and_service_check(dd_run_check, aggregator, instance): + check = ClaudeEnterpriseAnalyticsCheck("claude_enterprise_analytics", {}, [instance]) + + stub_client = mock.MagicMock() + stub_client.summaries.return_value = { + "daily_active_user_count": 95, + "weekly_active_user_count": 135, + "monthly_active_user_count": 143, + "assigned_seat_count": 150, + "pending_invite_count": 0, + "daily_adoption_rate": 63.33, + "weekly_adoption_rate": 90.0, + "monthly_adoption_rate": 95.33, + } + for name in ("users", "usage_report", "cost_report", "user_usage_report", "user_cost_report"): + getattr(stub_client, name).return_value = iter([]) + + with mock.patch( + "datadog_checks.claude_enterprise_analytics.check.AnthropicAnalyticsClient", + return_value=stub_client, + ): + dd_run_check(check) + + aggregator.assert_metric("claude_enterprise_analytics.org.dau", value=95) + aggregator.assert_metric("claude_enterprise_analytics.org.seats_assigned", value=150) + aggregator.assert_service_check("claude_enterprise_analytics.can_connect", status=AgentCheck.OK) + + +def test_check_emits_critical_service_check_on_api_error(aggregator, instance): + """When the client raises, the check should submit a CRITICAL service check + and re-raise so the Agent surfaces the failure.""" + check = ClaudeEnterpriseAnalyticsCheck("claude_enterprise_analytics", {}, [instance]) + + stub_client = mock.MagicMock() + stub_client.summaries.side_effect = RuntimeError("upstream 500") + + with mock.patch( + "datadog_checks.claude_enterprise_analytics.check.AnthropicAnalyticsClient", + return_value=stub_client, + ): + with pytest.raises(RuntimeError): + check.check({}) + + aggregator.assert_service_check("claude_enterprise_analytics.can_connect", status=AgentCheck.CRITICAL) + + +def test_check_merges_instance_tags_into_emitted_metrics(dd_run_check, aggregator): + """Custom `tags:` set on the instance must appear alongside the report_date + and source tags on every gauge. Uses `team`/`region` because ddev reserves + generic keys like `env`/`host`/`service`.""" + instance = { + "anthropic_api_key": "k", + "org_id": "test-org", + "lag_days": 3, + "tags": ["team:platform", "region:eu-west-1"], + } + check = ClaudeEnterpriseAnalyticsCheck("claude_enterprise_analytics", {}, [instance]) + + stub_client = mock.MagicMock() + stub_client.summaries.return_value = {"daily_active_user_count": 1} + for name in ("users", "usage_report", "cost_report", "user_usage_report", "user_cost_report"): + getattr(stub_client, name).return_value = iter([]) + + with mock.patch( + "datadog_checks.claude_enterprise_analytics.check.AnthropicAnalyticsClient", + return_value=stub_client, + ): + dd_run_check(check) + + aggregator.assert_metric_has_tag("claude_enterprise_analytics.org.dau", "team:platform") + aggregator.assert_metric_has_tag("claude_enterprise_analytics.org.dau", "region:eu-west-1") + aggregator.assert_metric_has_tag("claude_enterprise_analytics.org.dau", "org_id:test-org") + + +def test_check_requires_api_key(): + with pytest.raises(ConfigurationError): + ClaudeEnterpriseAnalyticsCheck("claude_enterprise_analytics", {}, [{}])