web search is billing $10 per 1,000 searches unnoticed

"""Report the per-search tool fee Claude web search is adding to the bill.

Read only. GET requests and nothing else: ANTHROPIC_ADMIN_KEY must be an Admin
API key (sk-ant-admin...), which can be provisioned read-only. A workspace key
is rejected by every /v1/organizations/* path.

This fee is not a token price. Web search bills $10 per 1,000 searches on top of
whatever the tokens cost, so no graph built on input and output tokens can show
it however carefully it is drawn.

The repair is printed, never applied. A max_uses cap and an allowed_domains
narrowing change what the agent is able to answer, and that belongs to whoever
owns the product, not to an audit holding an admin key.
"""
import argparse
import datetime as dt
import logging
import os
import sys

import requests

logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s")
log = logging.getLogger("anthropic_web_search_spend_audit")

API = "https://api.anthropic.com/v1"
VERSION = "2023-06-01"

# $10 per 1,000 searches, charged per search regardless of how many results come
# back. The unit is in the name because the natural slip is to multiply by ten
# and quote a bill a thousand times too large.
FEE_PER_THOUSAND = 10.0

# The cost report's own name for the row. Money for a server tool does not
# arrive as a token_type; it arrives under its own cost_type.
COST_TYPE = "web_search"

FINDINGS = ("search-fee",)


def fold(pages):
    """Sum server tool invocations per API key. Pure.

    server_tool_use is a nested object sitting beside the token fields, not one
    of them. Walking the result flat finds nothing and reports an organization
    running a million searches a month as running none.

    Counters other than web_search_requests are kept under their own names
    rather than dropped. New server tools ship, and a script that sums only the
    field it was written for keeps printing the same reassuring number after the
    next billable one arrives.
    """
    out = {}
    for page in pages:
        for bucket in page.get("data") or []:
            for result in bucket.get("results") or []:
                key = str(result.get("api_key_id") or "unattributed")
                row = out.setdefault(key, {"web_search": 0, "other_tools": {}})
                use = result.get("server_tool_use")
                if not isinstance(use, dict):
                    continue
                for name, value in use.items():
                    try:
                        count = int(value or 0)
                    except (TypeError, ValueError):
                        continue
                    if count <= 0:
                        continue
                    if name == "web_search_requests":
                        row["web_search"] += count
                    else:
                        row["other_tools"][name] = row["other_tools"].get(name, 0) + count
    return out


def fee(searches, per_thousand=FEE_PER_THOUSAND):
    """Dollars owed for a number of searches. Pure."""
    try:
        n = int(searches or 0)
    except (TypeError, ValueError):
        return 0.0
    return max(0, n) * per_thousand / 1000.0


def search_spend(cost_buckets, cost_type=COST_TYPE):
    """Sum the cost report rows the platform itself calls web search. Pure.

    amount arrives as a decimal string, not a number. Summing the raw values
    concatenates them in one language and throws in the other, and the failure
    is quiet enough to ship.
    """
    total = 0.0
    for bucket in cost_buckets or []:
        for result in bucket.get("results") or []:
            if str(result.get("cost_type") or "") != cost_type:
                continue
            raw = result.get("amount")
            if raw is None or raw == "":
                continue
            try:
                total += float(raw)
            except (TypeError, ValueError):
                pass
    return total


def verdict(row, min_searches=100):
    """Classify one key's search volume. Pure. Returns (state, detail).

    The floor exists because a handful of searches is a demo, not a bill, and a
    finding printed against it costs the reader more attention than it saves
    them money.
    """
    searches = int((row or {}).get("web_search") or 0)
    if searches <= 0:
        return ("no-searches", "the web search tool was never invoked by this key")
    if searches < min_searches:
        return ("low-volume",
                "%d search(es), under the floor of %d, worth about $%.2f"
                % (searches, min_searches, fee(searches)))
    return ("search-fee",
            "%d search(es) at $%.0f per 1,000, a tool fee of about $%.2f before "
            "a single token is priced"
            % (searches, FEE_PER_THOUSAND, fee(searches)))


def reconcile(estimate, billed, tolerance=0.25):
    """Compare the estimate against what was actually charged. Pure.

    Four states, not two, because the ways these two numbers can disagree have
    different explanations. A search that errors is counted as a use and not
    billed, so the estimate may legitimately run ahead. The cost report also
    lags. Neither is a licence to present one number as the other.
    """
    if estimate <= 0 and billed <= 0:
        return ("no-searches", "no searches counted and no web_search row billed")
    if billed <= 0:
        return ("unpriced",
                "$%.2f of searches counted and no web_search row on the cost "
                "report. Either the report has not caught up with the window, "
                "or the searches errored and were never billed." % estimate)
    if estimate <= 0:
        return ("billed-without-count",
                "$%.2f billed as web_search with no searches counted. The two "
                "reports are not covering the same days." % billed)
    drift = abs(billed - estimate) / estimate
    if drift <= tolerance:
        return ("confirmed",
                "$%.2f billed against $%.2f estimated, within %.0f%%"
                % (billed, estimate, tolerance * 100))
    return ("mismatch",
            "$%.2f billed against $%.2f estimated, %.0f%% apart. Read the "
            "web_search rows directly before quoting either number."
            % (billed, estimate, drift * 100))


def get(session, path, params):
    r = session.get(API + path, params=params, timeout=60)
    if r.status_code in (401, 403):
        raise SystemExit("%d from Anthropic: /v1/organizations/* needs an Admin "
                         "API key (sk-ant-admin...), not a workspace key"
                         % r.status_code)
    r.raise_for_status()
    return r.json()


def pages(session, path, params):
    """Walk the paginated usage or cost report."""
    params = dict(params)
    while True:
        page = get(session, path, params)
        yield page
        if not page.get("has_more") or not page.get("next_page"):
            return
        params["page"] = page["next_page"]


def window_start(days):
    """Floor to midnight UTC: starting_at must sit on a bucket boundary."""
    now = dt.datetime.now(dt.timezone.utc)
    midnight = now.replace(hour=0, minute=0, second=0, microsecond=0)
    return (midnight - dt.timedelta(days=days)).strftime("%Y-%m-%dT%H:%M:%SZ")


def main():
    ap = argparse.ArgumentParser(description=__doc__)
    ap.add_argument("--days", type=int, default=30,
                    help="days of daily buckets to read (default 30)")
    ap.add_argument("--min-searches", type=int, default=100,
                    help="searches below which no claim is made (default 100)")
    ap.add_argument("--show-all", action="store_true",
                    help="also print keys that never used the tool")
    args = ap.parse_args()

    admin = os.environ.get("ANTHROPIC_ADMIN_KEY")
    if not admin:
        log.error("set ANTHROPIC_ADMIN_KEY to an Admin API key (sk-ant-admin...); "
                  "a workspace key cannot read /v1/organizations/*")
        return 2

    s = requests.Session()
    s.headers.update({"x-api-key": admin, "anthropic-version": VERSION})

    start = window_start(args.days)
    rows = fold(pages(s, "/organizations/usage_report/messages",
                      {"starting_at": start, "bucket_width": "1d",
                       "limit": min(args.days + 1, 31),
                       "group_by[]": ["api_key_id"]}))

    cost_buckets = []
    for page in pages(s, "/organizations/cost_report",
                      {"starting_at": start, "limit": 31,
                       "group_by[]": ["description"]}):
        cost_buckets.extend(page.get("data") or [])

    checked = 0
    bad = 0
    estimate = 0.0
    for key in sorted(rows, key=lambda k: -rows[k]["web_search"]):
        row = rows[key]
        state, detail = verdict(row, args.min_searches)
        checked += 1
        estimate += fee(row["web_search"])
        line = "%-14s %-14s %s" % (state, key, detail)

        if state in FINDINGS:
            bad += 1
            log.warning(line)
            log.warning("  repair: set max_uses on the web search tool "
                        "definition for this service and narrow allowed_domains "
                        "to the hosts its answers actually cite")
            log.warning("  note: search results also re-enter input tokens on "
                        "every later turn of the same conversation, which is a "
                        "second charge this fee does not include")
        elif state == "low-volume" or args.show_all:
            log.info(line)

        for name, count in sorted(row["other_tools"].items()):
            log.info("  other server tool %s: %d use(s) by %s", name, count, key)

    billed = search_spend(cost_buckets)
    state, detail = reconcile(estimate, billed)
    (log.info if state in ("confirmed", "no-searches") else log.warning)(
        "%-14s %s", state, detail)

    log.info("%d key(s) checked, %d finding(s)", checked, bad)
    return 1 if bad else 0


if __name__ == "__main__":
    sys.exit(main())

/**
 * Report the per-search tool fee Claude web search is adding to the bill.
 *
 * Read only. GET requests and nothing else against the Admin API, which needs
 * an Admin API key (sk-ant-admin...); a workspace key is rejected by every
 * /v1/organizations/* path, and an Admin key can be provisioned read-only.
 *
 * The fee is not a token price. Web search bills $10 per 1,000 searches on top
 * of the tokens, so no graph built on input and output tokens can show it. The
 * repair is printed, never applied.
 */
const API = 'https://api.anthropic.com/v1';
const VERSION = '2023-06-01';

// $10 per 1,000 searches, charged per search regardless of how many results
// come back. The unit is in the name because the natural slip is to multiply by
// ten and quote a bill a thousand times too large.
const FEE_PER_THOUSAND = 10.0;

// The cost report's own name for the row. Server tool money does not arrive as
// a token_type; it arrives under its own cost_type.
const COST_TYPE = 'web_search';

const FINDINGS = ['search-fee'];

/**
 * Sum server tool invocations per API key. Pure.
 *
 * server_tool_use is a nested object sitting beside the token fields, not one
 * of them. Walking the result flat finds nothing. Counters other than
 * web_search_requests are kept under their own names, because new server tools
 * ship and a script that sums one field keeps printing a reassuring number.
 */
export function fold(pages) {
  const out = {};
  for (const page of pages ?? []) {
    for (const bucket of page.data ?? []) {
      for (const result of bucket.results ?? []) {
        const key = String(result.api_key_id ?? 'unattributed');
        if (!out[key]) out[key] = { web_search: 0, other_tools: {} };
        const row = out[key];
        const use = result.server_tool_use;
        if (use === null || typeof use !== 'object' || Array.isArray(use)) continue;
        for (const [name, value] of Object.entries(use)) {
          const count = Math.trunc(Number(value ?? 0));
          if (!Number.isFinite(count) || count <= 0) continue;
          if (name === 'web_search_requests') row.web_search += count;
          else row.other_tools[name] = (row.other_tools[name] ?? 0) + count;
        }
      }
    }
  }
  return out;
}

/** Dollars owed for a number of searches. Pure. */
export function fee(searches, perThousand = FEE_PER_THOUSAND) {
  const n = Math.trunc(Number(searches ?? 0));
  if (!Number.isFinite(n)) return 0;
  return Math.max(0, n) * perThousand / 1000;
}

/**
 * Sum the cost report rows the platform itself calls web search. Pure.
 * amount arrives as a decimal string, not a number.
 */
export function searchSpend(costBuckets, costType = COST_TYPE) {
  let total = 0;
  for (const bucket of costBuckets ?? []) {
    for (const result of bucket.results ?? []) {
      if (String(result.cost_type ?? '') !== costType) continue;
      const raw = result.amount;
      if (raw === null || raw === undefined || raw === '') continue;
      const value = Number(raw);
      if (Number.isFinite(value)) total += value;
    }
  }
  return total;
}

/**
 * Classify one key's search volume. Pure. Returns [state, detail].
 * The floor exists because a handful of searches is a demo, not a bill.
 */
export function verdict(row, minSearches = 100) {
  const searches = Math.trunc(Number(row?.web_search ?? 0)) || 0;
  if (searches <= 0) {
    return ['no-searches', 'the web search tool was never invoked by this key'];
  }
  if (searches < minSearches) {
    return ['low-volume',
      `${searches} search(es), under the floor of ${minSearches}, worth about ` +
      `$${fee(searches).toFixed(2)}`];
  }
  return ['search-fee',
    `${searches} search(es) at $${FEE_PER_THOUSAND.toFixed(0)} per 1,000, a ` +
    `tool fee of about $${fee(searches).toFixed(2)} before a single token is priced`];
}

/**
 * Compare the estimate against what was actually charged. Pure.
 *
 * Four states, not two. An errored search is counted as a use and not billed,
 * so the estimate may legitimately run ahead, and the cost report also lags.
 * Neither is a licence to present one number as the other.
 */
export function reconcile(estimate, billed, tolerance = 0.25) {
  if (estimate <= 0 && billed <= 0) {
    return ['no-searches', 'no searches counted and no web_search row billed'];
  }
  if (billed <= 0) {
    return ['unpriced',
      `$${estimate.toFixed(2)} of searches counted and no web_search row on ` +
      'the cost report. Either the report has not caught up with the window, ' +
      'or the searches errored and were never billed.'];
  }
  if (estimate <= 0) {
    return ['billed-without-count',
      `$${billed.toFixed(2)} billed as web_search with no searches counted. ` +
      'The two reports are not covering the same days.'];
  }
  const drift = Math.abs(billed - estimate) / estimate;
  if (drift <= tolerance) {
    return ['confirmed',
      `$${billed.toFixed(2)} billed against $${estimate.toFixed(2)} estimated, ` +
      `within ${(tolerance * 100).toFixed(0)}%`];
  }
  return ['mismatch',
    `$${billed.toFixed(2)} billed against $${estimate.toFixed(2)} estimated, ` +
    `${(drift * 100).toFixed(0)}% apart. Read the web_search rows directly ` +
    'before quoting either number.'];
}

async function get(key, path, params) {
  const url = new URL(API + path);
  for (const [k, v] of Object.entries(params)) {
    if (Array.isArray(v)) for (const item of v) url.searchParams.append(k, String(item));
    else if (v !== undefined && v !== null) url.searchParams.set(k, String(v));
  }
  const res = await fetch(url, {
    headers: { 'x-api-key': key, 'anthropic-version': VERSION },
  });
  if (res.status === 401 || res.status === 403) {
    throw new Error(`${res.status} from Anthropic: /v1/organizations/* needs an ` +
                    'Admin API key (sk-ant-admin...), not a workspace key');
  }
  if (!res.ok) throw new Error(`${res.status} from ${path}`);
  return res.json();
}

async function readPages(key, path, params) {
  const out = [];
  let next = { ...params };
  for (;;) {
    const page = await get(key, path, next);
    out.push(page);
    if (!page.has_more || !page.next_page) return out;
    next = { ...next, page: page.next_page };
  }
}

/** Floor to midnight UTC: starting_at must sit on a bucket boundary. */
function windowStart(days) {
  const midnight = new Date();
  midnight.setUTCHours(0, 0, 0, 0);
  midnight.setUTCDate(midnight.getUTCDate() - days);
  return midnight.toISOString().replace(/\.\d{3}Z$/, 'Z');
}

async function main() {
  const key = process.env.ANTHROPIC_ADMIN_KEY;
  if (!key) {
    console.error('set ANTHROPIC_ADMIN_KEY to an Admin API key (sk-ant-admin...); ' +
                  'a workspace key cannot read /v1/organizations/*');
    process.exitCode = 2;
    return;
  }

  const days = Number(process.env.DAYS ?? 30);
  const minSearches = Number(process.env.MIN_SEARCHES ?? 100);
  const showAll = process.argv.includes('--show-all');
  const start = windowStart(days);

  const usage = await readPages(key, '/organizations/usage_report/messages', {
    starting_at: start, bucket_width: '1d', limit: Math.min(days + 1, 31),
    'group_by[]': ['api_key_id'],
  });
  const rows = fold(usage);

  const costBuckets = [];
  for (const page of await readPages(key, '/organizations/cost_report',
    { starting_at: start, limit: 31, 'group_by[]': ['description'] })) {
    costBuckets.push(...(page.data ?? []));
  }

  let checked = 0;
  let bad = 0;
  let estimate = 0;
  const keys = Object.keys(rows).sort((a, b) => rows[b].web_search - rows[a].web_search);
  for (const id of keys) {
    const row = rows[id];
    const [state, detail] = verdict(row, minSearches);
    checked += 1;
    estimate += fee(row.web_search);
    const line = `${state.padEnd(14)} ${id.padEnd(14)} ${detail}`;

    if (FINDINGS.includes(state)) {
      bad += 1;
      console.warn(line);
      console.warn('  repair: set max_uses on the web search tool definition ' +
                   'for this service and narrow allowed_domains to the hosts ' +
                   'its answers actually cite');
      console.warn('  note: search results also re-enter input tokens on every ' +
                   'later turn of the same conversation, which is a second ' +
                   'charge this fee does not include');
    } else if (state === 'low-volume' || showAll) {
      console.log(line);
    }

    for (const name of Object.keys(row.other_tools).sort()) {
      console.log(`  other server tool ${name}: ${row.other_tools[name]} use(s) by ${id}`);
    }
  }

  const billed = searchSpend(costBuckets);
  const [state, detail] = reconcile(estimate, billed);
  const say = (state === 'confirmed' || state === 'no-searches') ? console.log : console.warn;
  say(`${state.padEnd(14)} ${detail}`);

  console.log(`${checked} key(s) checked, ${bad} finding(s)`);
  process.exitCode = bad ? 1 : 0;
}

// Only run when invoked directly, so importing this module from the test file
// does not fire main(), fail on the missing key, and set an exit code that
// fails the suite even as every test passes.
if (import.meta.url === `file://${process.argv[1]}`) {
  main().catch((err) => { console.error(err.message); process.exitCode = 2; });
}

from anthropic_web_search_spend_audit import (fee, fold, reconcile,
                                              search_spend, verdict)


def page(*results):
    """One page of GET /v1/organizations/usage_report/messages."""
    return {"data": [{"starting_at": "2026-08-01T00:00:00Z", "results": list(results)}],
            "has_more": False}


def usage(key="apikey_01Rs", searches=None, **tools):
    """One usage result. server_tool_use is nested beside the token fields."""
    use = dict(tools)
    if searches is not None:
        use["web_search_requests"] = searches
    row = {"api_key_id": key, "uncached_input_tokens": 900000,
           "output_tokens": 40000}
    if use:
        row["server_tool_use"] = use
    return row


def cost(cost_type="web_search", amount="1174.40"):
    """One bucket of GET /v1/organizations/cost_report."""
    return {"starting_at": "2026-08-01T00:00:00Z",
            "results": [{"cost_type": cost_type, "amount": amount,
                         "currency": "USD"}]}


def test_the_counter_is_nested_and_a_flat_read_finds_nothing():
    # The whole note. Both results carry a five figure search count, and a fold
    # that only looks at top-level fields reports this key as never searching.
    rows = fold([page(usage(searches=60000), usage(searches=58400))])
    assert rows["apikey_01Rs"]["web_search"] == 118400
    # A result with no server_tool_use at all is still a key, with zero searches.
    assert fold([page(usage())])["apikey_01Rs"]["web_search"] == 0


def test_the_fee_is_per_thousand_searches_not_per_search():
    assert fee(118400) == 1184.00
    assert fee(1) == 0.01
    assert fee(0) == 0.0
    assert fee(None) == 0.0


def test_a_high_volume_key_is_the_finding_and_quotes_the_fee():
    state, detail = verdict({"web_search": 118400, "other_tools": {}})
    assert state == "search-fee"
    assert "tool fee of about $1184.00" in detail


def test_a_handful_of_searches_is_a_demo_and_not_a_bill():
    assert verdict({"web_search": 12, "other_tools": {}})[0] == "low-volume"
    assert verdict({"web_search": 0, "other_tools": {}})[0] == "no-searches"
    assert verdict({})[0] == "no-searches"


def test_an_unknown_server_tool_counter_stays_visible():
    # A counter this script was not written for must not be silently dropped,
    # or the next billable server tool arrives and nothing changes on screen.
    rows = fold([page(usage(searches=200, web_fetch_requests=90,
                            code_execution_sessions=0))])
    row = rows["apikey_01Rs"]
    assert row["web_search"] == 200
    assert row["other_tools"] == {"web_fetch_requests": 90}


def test_only_the_web_search_cost_type_counts_and_amount_is_a_string():
    buckets = [cost("web_search", "1174.40"),
               cost("web_search", "10.00"),
               cost("code_execution", "500.00"),
               cost("web_search", "")]
    assert search_spend(buckets) == 1184.40
    assert search_spend([]) == 0.0


def test_the_four_ways_the_two_reports_can_disagree_stay_four_answers():
    assert reconcile(1184.00, 1174.40)[0] == "confirmed"
    # Counted but not billed: errored searches are free, and the report lags.
    assert reconcile(1184.00, 0.0)[0] == "unpriced"
    # Billed but not counted: the two windows do not line up.
    assert reconcile(0.0, 1174.40)[0] == "billed-without-count"
    state, detail = reconcile(100.00, 900.00)
    assert state == "mismatch"
    assert "800% apart" in detail
    assert reconcile(0.0, 0.0)[0] == "no-searches"

import { test } from 'node:test';
import assert from 'node:assert/strict';
import { fee, fold, reconcile, searchSpend, verdict }
  from './anthropic-web-search-spend-audit.mjs';

/** One page of GET /v1/organizations/usage_report/messages. */
function page(...results) {
  return { data: [{ starting_at: '2026-08-01T00:00:00Z', results }], has_more: false };
}

/** One usage result. server_tool_use is nested beside the token fields. */
function usage({ key = 'apikey_01Rs', searches = null, tools = {} } = {}) {
  const use = { ...tools };
  if (searches !== null) use.web_search_requests = searches;
  const row = { api_key_id: key, uncached_input_tokens: 900000, output_tokens: 40000 };
  if (Object.keys(use).length) row.server_tool_use = use;
  return row;
}

/** One bucket of GET /v1/organizations/cost_report. */
function cost(costType = 'web_search', amount = '1174.40') {
  return {
    starting_at: '2026-08-01T00:00:00Z',
    results: [{ cost_type: costType, amount, currency: 'USD' }],
  };
}

test('the counter is nested and a flat read finds nothing', () => {
  const rows = fold([page(usage({ searches: 60000 }), usage({ searches: 58400 }))]);
  assert.equal(rows.apikey_01Rs.web_search, 118400);
  assert.equal(fold([page(usage())]).apikey_01Rs.web_search, 0);
});

test('the fee is per thousand searches, not per search', () => {
  assert.equal(fee(118400), 1184.00);
  assert.equal(fee(1), 0.01);
  assert.equal(fee(0), 0);
  assert.equal(fee(null), 0);
});

test('a high volume key is the finding and quotes the fee', () => {
  const [state, detail] = verdict({ web_search: 118400, other_tools: {} });
  assert.equal(state, 'search-fee');
  assert.match(detail, /tool fee of about \$1184\.00/);
});

test('a handful of searches is a demo and not a bill', () => {
  assert.equal(verdict({ web_search: 12, other_tools: {} })[0], 'low-volume');
  assert.equal(verdict({ web_search: 0, other_tools: {} })[0], 'no-searches');
  assert.equal(verdict({})[0], 'no-searches');
});

test('an unknown server tool counter stays visible', () => {
  const rows = fold([page(usage({
    searches: 200,
    tools: { web_fetch_requests: 90, code_execution_sessions: 0 },
  }))]);
  assert.equal(rows.apikey_01Rs.web_search, 200);
  assert.deepEqual(rows.apikey_01Rs.other_tools, { web_fetch_requests: 90 });
});

test('only the web_search cost type counts and amount is a string', () => {
  const buckets = [cost('web_search', '1174.40'), cost('web_search', '10.00'),
                   cost('code_execution', '500.00'), cost('web_search', '')];
  assert.equal(searchSpend(buckets), 1184.40);
  assert.equal(searchSpend([]), 0);
});

test('the four ways the two reports can disagree stay four answers', () => {
  assert.equal(reconcile(1184.00, 1174.40)[0], 'confirmed');
  assert.equal(reconcile(1184.00, 0)[0], 'unpriced');
  assert.equal(reconcile(0, 1174.40)[0], 'billed-without-count');
  const [state, detail] = reconcile(100.00, 900.00);
  assert.equal(state, 'mismatch');
  assert.match(detail, /800% apart/);
  assert.equal(reconcile(0, 0)[0], 'no-searches');
});