Tool schemas are most of the input tokens on every call

"""Measure what a Claude tools block costs in input tokens on every call.

Read only. One GET for the model object and a handful of calls to
/v1/messages/count_tokens, which is free, creates no object, generates no
completion and is not billed. /v1/messages is never called.

The method is subtraction: count the exact body, count it again with tools
removed, and the difference is the per-call tool overhead. Ablating one tool at
a time prices each schema, and the deltas deliberately do not sum to the whole,
because every ablated body still carries the automatic tool-use system prompt.

The repair is printed, never performed. A cache breakpoint is a deploy.
"""
import argparse
import copy
import json
import logging
import os
import sys

import requests

logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s")
log = logging.getLogger("anthropic_tool_schema_overhead")

API = "https://api.anthropic.com/v1"
VERSION = "2023-06-01"

# Fields the counting endpoint refuses. Stripped from every body before it is
# counted, and stripped identically from all of them so the subtraction stays
# honest: a field removed from one body and not another moves the difference.
SAMPLING_ONLY = ("max_tokens", "stream", "temperature", "top_p", "top_k",
                 "stop_sequences", "metadata", "service_tier")

# The automatic tool-use system prompt, per model, as (auto_or_none,
# any_or_tool). Added by the API whenever any tool is present, so it is part of
# the overhead and no amount of pruning removes it. Matched on longest prefix:
# a substring test reads claude-opus-4-5 as claude-opus-5 and reports the wrong
# fixed charge with total confidence.
TOOL_SYSTEM_PROMPT = {
    "claude-opus-5": (286, 406),
    "claude-opus-4-8": (290, 410),
    "claude-opus-4-7": (675, 804),
    "claude-opus-4-6": (497, 589),
    "claude-sonnet-4-6": (497, 589),
    "claude-sonnet-5": (354, 474),
    "claude-opus-4-5": (496, 588),
    "claude-sonnet-4-5": (496, 588),
    "claude-haiku-4-5": (496, 588),
}

FINDINGS = ("schema-dominates", "schema-heavy")


def _int(value):
    """Read a token count as an int. Pure. Missing and unreadable both mean 0."""
    try:
        return int(value or 0)
    except (TypeError, ValueError):
        return 0


def countable(body):
    """A body the counting endpoint will accept. Pure. Does not mutate.

    Only the sampling fields go. Everything being measured stays, because the
    number is worthless if the thing counted is not the thing sent.
    """
    if not isinstance(body, dict):
        return {}
    return {k: copy.deepcopy(v) for k, v in body.items() if k not in SAMPLING_ONLY}


def without_tools(body):
    """The same body with the whole tools block removed. Pure.

    tool_choice goes with it. A body that names a tool it no longer declares is
    rejected, and the rejection would be read as "the counter is broken".
    """
    stripped = countable(body)
    stripped.pop("tools", None)
    stripped.pop("tool_choice", None)
    return stripped


def tool_names(body):
    """Named tools in a body, in declaration order. Pure."""
    out = []
    for tool in (body or {}).get("tools") or []:
        if not isinstance(tool, dict):
            continue
        name = str(tool.get("name") or "").strip()
        if name and name not in out:
            out.append(name)
    return out


def without_tool(body, name):
    """The same body with exactly one tool removed. Pure. Does not mutate."""
    stripped = countable(body)
    kept = [t for t in stripped.get("tools") or []
            if not (isinstance(t, dict) and str(t.get("name") or "") == str(name))]
    stripped["tools"] = kept
    if not kept:
        stripped.pop("tools", None)
        stripped.pop("tool_choice", None)
    return stripped


def overhead(total, base):
    """Tokens attributable to the tools block. Pure. Never negative."""
    return max(0, _int(total) - _int(base))


def overhead_share(total, base):
    """Share of the counted input that the tools block accounts for. Pure.

    None when nothing was counted, which is a different state from zero and
    must not be printed as 0%.
    """
    counted = _int(total)
    if counted <= 0:
        return None
    return overhead(total, base) / float(counted)


def choice_kind(body):
    """Which column of the tool-use system prompt table applies. Pure.

    auto and none share one size; any and a named tool share the larger one.
    """
    choice = (body or {}).get("tool_choice")
    kind = ""
    if isinstance(choice, str):
        kind = choice.strip().lower()
    elif isinstance(choice, dict):
        kind = str(choice.get("type") or "").strip().lower()
    if kind in ("any", "tool"):
        return "any"
    return "auto"


def system_prompt_tokens(model, kind="auto"):
    """The automatic tool-use system prompt for one model. Pure. None if unlisted.

    Longest prefix wins. Unlisted ids return None rather than a neighbour's
    number, because a plausible wrong number here silently corrupts the split
    between "your schemas" and "the fixed charge".
    """
    name = str(model or "").strip().lower()
    best = None
    best_len = -1
    for prefix, sizes in TOOL_SYSTEM_PROMPT.items():
        if (name == prefix or name.startswith(prefix + "-")) and len(prefix) > best_len:
            best = sizes
            best_len = len(prefix)
    if best is None:
        return None
    return best[1] if str(kind).lower() == "any" else best[0]


def fixed_overhead(total_overhead, per_tool):
    """The part of the tool overhead that belongs to no single tool. Pure.

    Ablating one tool never removes the automatic tool-use system prompt,
    because the remaining tools still require it. So the per-tool deltas sum to
    the schema weight alone and the residual is the fixed charge for having any
    tools at all. Printing the sum as if it were the total is the mistake this
    function exists to make impossible.
    """
    measured = sum(max(0, _int(row.get("tokens"))) for row in per_tool or [])
    return max(0, _int(total_overhead) - measured), measured


def classify(total, base, dominate=0.5, heavy=0.25):
    """Classify one measured payload. Pure. Returns (state, detail)."""
    counted = _int(total)
    if counted <= 0:
        return ("nothing-counted",
                "the counting endpoint returned no tokens for this body")
    weight = overhead(total, base)
    if weight <= 0:
        return ("no-tools",
                "%d input token(s) and no measurable tools block" % counted)
    share = weight / float(counted)
    rest = counted - weight
    shape = ("%d of %d input token(s) are the tools block (%.0f%%)"
             % (weight, counted, share * 100))
    if rest > 0:
        shape += (", against %d token(s) of system and messages, a ratio of "
                  "%.1f to 1" % (rest, weight / float(rest)))
    if share >= dominate:
        return ("schema-dominates",
                shape + ". The machinery outweighs the conversation on every "
                "call, cached or not.")
    if share >= heavy:
        return ("schema-heavy",
                shape + ". Not dominant, and still the single largest stable "
                "block in the prompt, which makes it the cheapest thing to "
                "cache.")
    return ("schema-modest", shape + ".")


def defer_candidates(rows, hot=(), keep_eager=1):
    """Tools that could carry defer_loading, and never all of them. Pure.

    The API answers a request whose every tool defers with 400, "All tools have
    defer_loading set". A function able to return the whole list is a function
    that has already caused an outage, so at least one tool always stays eager
    whatever the arithmetic says.
    """
    names = [str(r.get("name")) for r in rows or [] if r.get("name")]
    if len(names) <= keep_eager:
        return []
    hot_set = {str(h) for h in hot or []}
    candidates = [n for n in names if n not in hot_set]
    if len(candidates) >= len(names):
        heaviest = sorted(rows, key=lambda r: -_int(r.get("tokens")))
        eager = {str(r.get("name")) for r in heaviest[:max(1, keep_eager)]}
        candidates = [n for n in names if n not in eager]
    return candidates


def monthly_cost(tokens_per_call, calls_per_day, rate_per_mtok, days=30):
    """What one per-call token count costs in a month. Pure. None if unpriced."""
    tokens = _int(tokens_per_call)
    calls = _int(calls_per_day)
    try:
        rate = float(rate_per_mtok)
    except (TypeError, ValueError):
        return None
    if tokens <= 0 or calls <= 0 or rate <= 0:
        return None
    return tokens * calls * int(days) / 1_000_000.0 * rate


def window_share(total, window):
    """Share of the model context window spent before the user speaks. Pure."""
    size = _int(window)
    if size <= 0:
        return None
    return min(1.0, _int(total) / float(size))


def get(session, path):
    r = session.get(API + path, timeout=30)
    if r.status_code in (401, 403):
        raise SystemExit("%d from Anthropic: ANTHROPIC_API_KEY has to be a "
                         "workspace key" % r.status_code)
    if r.status_code == 404:
        return {}
    r.raise_for_status()
    return r.json()


def count(session, body):
    """The one non-GET call. It creates nothing, generates nothing, bills nothing."""
    r = session.post(API + "/messages/count_tokens", json=body, timeout=120)
    if r.status_code >= 400:
        log.warning("count_tokens answered %d: %s", r.status_code, r.text[:200])
        return None
    return _int((r.json() or {}).get("input_tokens"))


def main():
    ap = argparse.ArgumentParser(description=__doc__)
    ap.add_argument("--payload", action="append", default=[], required=True,
                    metavar="FILE", help="a JSON file holding a real request body")
    ap.add_argument("--calls-per-day", type=int, default=10000,
                    help="calls of this shape per day, for the monthly price")
    ap.add_argument("--input-rate", type=float, default=3.0,
                    help="your model's uncached input rate per million tokens")
    ap.add_argument("--hot", action="append", default=[],
                    help="a tool name that must stay eagerly loaded; repeatable")
    ap.add_argument("--no-per-tool", action="store_true",
                    help="skip the per-tool ablation")
    args = ap.parse_args()

    key = os.environ.get("ANTHROPIC_API_KEY")
    if not key:
        log.error("set ANTHROPIC_API_KEY to a workspace key")
        return 2

    session = requests.Session()
    session.headers.update({"x-api-key": key, "anthropic-version": VERSION,
                            "content-type": "application/json"})

    checked = 0
    bad = 0
    for path in args.payload:
        with open(path, "r", encoding="utf-8") as fh:
            body = json.load(fh)
        checked += 1

        total = count(session, countable(body))
        base = count(session, without_tools(body))
        if total is None or base is None:
            log.warning("could not measure %s", path)
            continue

        state, detail = classify(total, base)
        line = "%-18s %-24s %s" % (state, path, detail)
        if state in FINDINGS:
            bad += 1
            log.warning(line)
        else:
            log.info(line)

        model = str(body.get("model") or "")
        kind = choice_kind(body)
        fixed = system_prompt_tokens(model, kind)
        if fixed is None:
            log.info("  no published tool-use system prompt size for %r, so "
                     "the fixed charge cannot be separated out here", model)
        else:
            log.info("  %d of the overhead is the automatic tool-use system "
                     "prompt for %s at tool_choice %s", fixed, model, kind)

        rows = []
        if not args.no_per_tool:
            for name in tool_names(body):
                one = count(session, without_tool(body, name))
                if one is None:
                    continue
                rows.append({"name": name, "tokens": max(0, total - one)})
            rows.sort(key=lambda r: -r["tokens"])
            residual, measured = fixed_overhead(overhead(total, base), rows)
            log.info("  the fixed charge no ablation removes: %d token(s); "
                     "your schemas account for %d", residual, measured)
            if rows:
                log.info("  heaviest: %s", ", ".join(
                    "%s %d" % (r["name"], r["tokens"]) for r in rows[:3]))

        window = get(session, "/models/" + model).get("max_input_tokens") if model else None
        share = window_share(total, window)
        if share is not None:
            log.info("  %.0f%% of the %d token context window is spent before "
                     "the user says anything. Whether a real conversation still "
                     "fits is the context-overflow question, not this one.",
                     share * 100, _int(window))

        price = monthly_cost(overhead(total, base), args.calls_per_day,
                             args.input_rate)
        if price is not None:
            log.info("  at %d call(s) a day and %.2f per million input tokens "
                     "that is %.2f a month, uncached", args.calls_per_day,
                     args.input_rate, price)

        if state in FINDINGS:
            log.warning("  repair: put a cache_control breakpoint after the "
                        "tools block. A read costs 0.1x base input, and tools "
                        "are the most stable thing in the prompt.")
            log.warning("  repair: editing any tool description after that "
                        "invalidates the tools, the system prompt and the "
                        "messages behind them. Batch tool edits.")
            candidates = defer_candidates(rows, args.hot)
            if candidates:
                log.warning("  repair: defer_loading on rarely used tools only "
                            "(%s). Never on all of them: the API answers 400, "
                            "All tools have defer_loading set. Which are rare "
                            "is a call-coverage question this script cannot "
                            "answer.", ", ".join(candidates[:5]))

    log.info("%d payload(s) measured, %d finding(s)", checked, bad)
    return 1 if bad else 0


if __name__ == "__main__":
    sys.exit(main())

/**
 * Measure what a Claude tools block costs in input tokens on every call.
 *
 * Read only. One GET for the model object and a handful of calls to
 * /v1/messages/count_tokens, which is free, creates no object, generates no
 * completion and is not billed. /v1/messages is never called.
 *
 * Count the body, count it again with tools removed, subtract. Ablate one tool
 * at a time for a per-tool price, and note that the deltas do not sum to the
 * whole: every ablated body still carries the tool-use system prompt.
 */
import { readFile } from 'node:fs/promises';

const API = 'https://api.anthropic.com/v1';
const VERSION = '2023-06-01';

// Fields the counting endpoint refuses, stripped identically from every body.
const SAMPLING_ONLY = new Set(['max_tokens', 'stream', 'temperature', 'top_p',
  'top_k', 'stop_sequences', 'metadata', 'service_tier']);

// The automatic tool-use system prompt per model, as [autoOrNone, anyOrTool].
// Longest prefix wins: a substring test reads claude-opus-4-5 as claude-opus-5.
const TOOL_SYSTEM_PROMPT = {
  'claude-opus-5': [286, 406],
  'claude-opus-4-8': [290, 410],
  'claude-opus-4-7': [675, 804],
  'claude-opus-4-6': [497, 589],
  'claude-sonnet-4-6': [497, 589],
  'claude-sonnet-5': [354, 474],
  'claude-opus-4-5': [496, 588],
  'claude-sonnet-4-5': [496, 588],
  'claude-haiku-4-5': [496, 588],
};

const FINDINGS = new Set(['schema-dominates', 'schema-heavy']);

/** Read a token count as an integer. Pure. Missing and unreadable both mean 0. */
export function readInt(value) {
  const n = Number(value ?? 0);
  return Number.isFinite(n) ? Math.trunc(n) : 0;
}

/** A body the counting endpoint will accept. Pure. Does not mutate. */
export function countable(body) {
  if (!body || typeof body !== 'object') return {};
  const out = {};
  for (const [k, v] of Object.entries(body)) {
    if (!SAMPLING_ONLY.has(k)) out[k] = structuredClone(v);
  }
  return out;
}

/**
 * The same body with the whole tools block removed. Pure.
 * tool_choice goes with it: a body naming a tool it no longer declares is
 * rejected, and the rejection reads as a broken counter.
 */
export function withoutTools(body) {
  const stripped = countable(body);
  delete stripped.tools;
  delete stripped.tool_choice;
  return stripped;
}

/** Named tools in a body, in declaration order. Pure. */
export function toolNames(body) {
  const out = [];
  for (const tool of body?.tools ?? []) {
    if (!tool || typeof tool !== 'object') continue;
    const name = String(tool.name ?? '').trim();
    if (name && !out.includes(name)) out.push(name);
  }
  return out;
}

/** The same body with exactly one tool removed. Pure. Does not mutate. */
export function withoutTool(body, name) {
  const stripped = countable(body);
  const kept = (stripped.tools ?? []).filter(
    (t) => !(t && typeof t === 'object' && String(t.name ?? '') === String(name)));
  stripped.tools = kept;
  if (kept.length === 0) {
    delete stripped.tools;
    delete stripped.tool_choice;
  }
  return stripped;
}

/** Tokens attributable to the tools block. Pure. Never negative. */
export function overhead(total, base) {
  return Math.max(0, readInt(total) - readInt(base));
}

/** Share of counted input the tools block accounts for. Pure. Null when none. */
export function overheadShare(total, base) {
  const counted = readInt(total);
  if (counted <= 0) return null;
  return overhead(total, base) / counted;
}

/** Which column of the tool-use system prompt table applies. Pure. */
export function choiceKind(body) {
  const choice = body?.tool_choice;
  let kind = '';
  if (typeof choice === 'string') kind = choice.trim().toLowerCase();
  else if (choice && typeof choice === 'object') {
    kind = String(choice.type ?? '').trim().toLowerCase();
  }
  return kind === 'any' || kind === 'tool' ? 'any' : 'auto';
}

/**
 * The automatic tool-use system prompt for one model. Pure. Null if unlisted.
 * Unlisted returns null rather than a neighbour's number: a plausible wrong
 * value here silently corrupts the split between schemas and fixed charge.
 */
export function systemPromptTokens(model, kind = 'auto') {
  const name = String(model ?? '').trim().toLowerCase();
  let best = null;
  let bestLen = -1;
  for (const [prefix, sizes] of Object.entries(TOOL_SYSTEM_PROMPT)) {
    if ((name === prefix || name.startsWith(`${prefix}-`)) && prefix.length > bestLen) {
      best = sizes;
      bestLen = prefix.length;
    }
  }
  if (best === null) return null;
  return String(kind).toLowerCase() === 'any' ? best[1] : best[0];
}

/**
 * The part of the tool overhead that belongs to no single tool. Pure.
 * Returns [residual, measured]. Ablation never removes the tool-use system
 * prompt, so the deltas sum to the schema weight and the rest is fixed.
 */
export function fixedOverhead(totalOverhead, perTool) {
  let measured = 0;
  for (const row of perTool ?? []) measured += Math.max(0, readInt(row?.tokens));
  return [Math.max(0, readInt(totalOverhead) - measured), measured];
}

/** Classify one measured payload. Pure. Returns [state, detail]. */
export function classify(total, base, dominate = 0.5, heavy = 0.25) {
  const counted = readInt(total);
  if (counted <= 0) {
    return ['nothing-counted', 'the counting endpoint returned no tokens for this body'];
  }
  const weight = overhead(total, base);
  if (weight <= 0) {
    return ['no-tools', `${counted} input token(s) and no measurable tools block`];
  }
  const share = weight / counted;
  const rest = counted - weight;
  let shape = `${weight} of ${counted} input token(s) are the tools block ` +
    `(${(share * 100).toFixed(0)}%)`;
  if (rest > 0) {
    shape += `, against ${rest} token(s) of system and messages, a ratio of ` +
      `${(weight / rest).toFixed(1)} to 1`;
  }
  if (share >= dominate) {
    return ['schema-dominates',
      `${shape}. The machinery outweighs the conversation on every call, ` +
      'cached or not.'];
  }
  if (share >= heavy) {
    return ['schema-heavy',
      `${shape}. Not dominant, and still the single largest stable block in ` +
      'the prompt, which makes it the cheapest thing to cache.'];
  }
  return ['schema-modest', `${shape}.`];
}

/**
 * Tools that could carry defer_loading, and never all of them. Pure.
 * The API answers a fully deferred request with 400, "All tools have
 * defer_loading set", so at least one tool always stays eager.
 */
export function deferCandidates(rows, hot = [], keepEager = 1) {
  const names = (rows ?? []).map((r) => String(r?.name ?? '')).filter(Boolean);
  if (names.length <= keepEager) return [];
  const hotSet = new Set((hot ?? []).map(String));
  let candidates = names.filter((n) => !hotSet.has(n));
  if (candidates.length >= names.length) {
    const heaviest = [...rows].sort((a, b) => readInt(b?.tokens) - readInt(a?.tokens));
    const eager = new Set(heaviest.slice(0, Math.max(1, keepEager))
      .map((r) => String(r?.name ?? '')));
    candidates = names.filter((n) => !eager.has(n));
  }
  return candidates;
}

/** What one per-call token count costs in a month. Pure. Null if unpriced. */
export function monthlyCost(tokensPerCall, callsPerDay, ratePerMtok, days = 30) {
  const tokens = readInt(tokensPerCall);
  const calls = readInt(callsPerDay);
  const rate = Number(ratePerMtok);
  if (!Number.isFinite(rate) || tokens <= 0 || calls <= 0 || rate <= 0) return null;
  return (tokens * calls * Math.trunc(days)) / 1000000 * rate;
}

/** Share of the model context window spent before the user speaks. Pure. */
export function windowShare(total, window) {
  const size = readInt(window);
  if (size <= 0) return null;
  return Math.min(1, readInt(total) / size);
}

function headers(key) {
  return { 'x-api-key': key, 'anthropic-version': VERSION,
           'content-type': 'application/json' };
}

async function get(key, path) {
  const res = await fetch(API + path, { headers: headers(key) });
  if (res.status === 401 || res.status === 403) {
    throw new Error(`${res.status} from Anthropic: ANTHROPIC_API_KEY has to be a workspace key`);
  }
  if (res.status === 404) return {};
  if (!res.ok) throw new Error(`${res.status} from ${path}`);
  return res.json();
}

/** The one non-GET call. It creates nothing, generates nothing, bills nothing. */
async function count(key, body) {
  const res = await fetch(`${API}/messages/count_tokens`, {
    method: 'POST',  // count_tokens creates nothing and bills nothing
    headers: headers(key),
    body: JSON.stringify(body),
  });
  if (!res.ok) {
    console.warn(`count_tokens answered ${res.status}`);
    return null;
  }
  return readInt((await res.json())?.input_tokens);
}

async function main() {
  const key = process.env.ANTHROPIC_API_KEY;
  if (!key) {
    console.error('set ANTHROPIC_API_KEY to a workspace key');
    process.exitCode = 2;
    return;
  }
  const paths = process.argv.slice(2).filter((a) => !a.startsWith('--'));
  if (paths.length === 0) {
    console.error('pass one or more payload JSON files');
    process.exitCode = 2;
    return;
  }
  const callsPerDay = Number(process.env.CALLS_PER_DAY ?? 10000);
  const inputRate = Number(process.env.INPUT_RATE ?? 3.0);
  const hot = String(process.env.HOT ?? '').split(',').filter(Boolean);
  const perTool = process.env.NO_PER_TOOL !== '1';

  let checked = 0;
  let bad = 0;
  for (const path of paths) {
    const body = JSON.parse(await readFile(path, 'utf8'));
    checked += 1;

    const total = await count(key, countable(body));
    const base = await count(key, withoutTools(body));
    if (total === null || base === null) {
      console.warn(`could not measure ${path}`);
      continue;
    }

    const [state, detail] = classify(total, base);
    const line = `${state.padEnd(18)} ${path.padEnd(24)} ${detail}`;
    if (FINDINGS.has(state)) {
      bad += 1;
      console.warn(line);
    } else {
      console.log(line);
    }

    const model = String(body.model ?? '');
    const kind = choiceKind(body);
    const fixed = systemPromptTokens(model, kind);
    if (fixed === null) {
      console.log(`  no published tool-use system prompt size for ${model}, so ` +
                  'the fixed charge cannot be separated out here');
    } else {
      console.log(`  ${fixed} of the overhead is the automatic tool-use system ` +
                  `prompt for ${model} at tool_choice ${kind}`);
    }

    let rows = [];
    if (perTool) {
      for (const name of toolNames(body)) {
        const one = await count(key, withoutTool(body, name));
        if (one === null) continue;
        rows.push({ name, tokens: Math.max(0, total - one) });
      }
      rows.sort((a, b) => b.tokens - a.tokens);
      const [residual, measured] = fixedOverhead(overhead(total, base), rows);
      console.log(`  the fixed charge no ablation removes: ${residual} token(s); ` +
                  `your schemas account for ${measured}`);
      if (rows.length > 0) {
        console.log(`  heaviest: ${rows.slice(0, 3)
          .map((r) => `${r.name} ${r.tokens}`).join(', ')}`);
      }
    }

    const window = model ? (await get(key, `/models/${model}`))?.max_input_tokens : null;
    const share = windowShare(total, window);
    if (share !== null) {
      console.log(`  ${(share * 100).toFixed(0)}% of the ${readInt(window)} token ` +
                  'context window is spent before the user says anything. Whether ' +
                  'a real conversation still fits is the context-overflow ' +
                  'question, not this one.');
    }

    const price = monthlyCost(overhead(total, base), callsPerDay, inputRate);
    if (price !== null) {
      console.log(`  at ${callsPerDay} call(s) a day and ${inputRate.toFixed(2)} ` +
                  `per million input tokens that is ${price.toFixed(2)} a month, uncached`);
    }

    if (FINDINGS.has(state)) {
      console.warn('  repair: put a cache_control breakpoint after the tools ' +
                   'block. A read costs 0.1x base input, and tools are the most ' +
                   'stable thing in the prompt.');
      console.warn('  repair: editing any tool description after that ' +
                   'invalidates the tools, the system prompt and the messages ' +
                   'behind them. Batch tool edits.');
      const candidates = deferCandidates(rows, hot);
      if (candidates.length > 0) {
        console.warn(`  repair: defer_loading on rarely used tools only ` +
                     `(${candidates.slice(0, 5).join(', ')}). Never on all of ` +
                     'them: the API answers 400, All tools have defer_loading ' +
                     'set. Which are rare is a call-coverage question this ' +
                     'script cannot answer.');
      }
    }
  }

  console.log(`${checked} payload(s) measured, ${bad} finding(s)`);
  process.exitCode = bad ? 1 : 0;
}

if (import.meta.url === `file://${process.argv[1]}`) {
  main().catch((err) => { console.error(err.message); process.exitCode = 2; });
}

from anthropic_tool_schema_overhead import (choice_kind, classify, countable,
                                            defer_candidates, fixed_overhead,
                                            monthly_cost, overhead,
                                            overhead_share,
                                            system_prompt_tokens, tool_names,
                                            window_share, without_tool,
                                            without_tools)

BODY = {
    "model": "claude-opus-5",
    "max_tokens": 1024,
    "temperature": 0,
    "system": "You are a support agent.",
    "tool_choice": {"type": "auto"},
    "messages": [{"role": "user", "content": "where is my order"}],
    "tools": [
        {"name": "search_knowledge_base", "input_schema": {"type": "object"}},
        {"name": "create_ticket", "input_schema": {"type": "object"}},
        {"name": "lookup_order", "input_schema": {"type": "object"}},
    ],
}


def test_the_tools_block_is_most_of_what_you_pay_for():
    # The note in one assertion. Two free counts, one subtraction.
    total, base = 12388, 888
    assert overhead(total, base) == 11500
    assert round(overhead_share(total, base), 4) == 0.9283

    state, detail = classify(total, base)
    assert state == "schema-dominates"
    assert "11500 of 12388 input token(s) are the tools block (93%)" in detail
    assert "888 token(s) of system and messages, a ratio of 13.0 to 1" in detail

    # 11500 tokens on 10000 calls a day for 30 days at $3 per million.
    assert monthly_cost(11500, 10000, 3.0) == 10350.0


def test_the_ablation_deltas_do_not_add_up_to_the_whole():
    # The trap. Removing one tool never removes the tool-use system prompt, so
    # the per-tool sum is the schema weight and the residual is the fixed
    # charge. A script that printed the sum as the total would be wrong by 286
    # tokens on every call and would look right.
    per_tool = [{"name": "search_knowledge_base", "tokens": 6200},
                {"name": "create_ticket", "tokens": 3100},
                {"name": "lookup_order", "tokens": 1914}]
    residual, measured = fixed_overhead(11500, per_tool)
    assert measured == 11214
    assert residual == 286
    assert residual == system_prompt_tokens("claude-opus-5", "auto")
    assert fixed_overhead(0, per_tool) == (0, 11214)


def test_the_system_prompt_table_matches_on_longest_prefix():
    assert system_prompt_tokens("claude-opus-5") == 286
    assert system_prompt_tokens("claude-opus-5", "any") == 406
    assert system_prompt_tokens("claude-sonnet-5") == 354
    # The one a careless substring match gets wrong: 4-5 is not 5.
    assert system_prompt_tokens("claude-opus-4-5") == 496
    assert system_prompt_tokens("claude-haiku-4-5-20251001") == 496
    assert system_prompt_tokens("claude-opus-4-7", "any") == 804
    # Unlisted returns nothing rather than a neighbour's number.
    assert system_prompt_tokens("claude-fable-5") is None
    assert system_prompt_tokens("") is None
    assert system_prompt_tokens(None) is None


def test_removing_the_tools_removes_the_tool_choice_with_them():
    stripped = without_tools(BODY)
    assert "tools" not in stripped and "tool_choice" not in stripped
    assert stripped["system"] == BODY["system"]
    assert stripped["messages"] == BODY["messages"]
    # And the original is untouched, or the second count measures the first.
    assert len(BODY["tools"]) == 3 and "tool_choice" in BODY

    one_out = without_tool(BODY, "create_ticket")
    assert tool_names(one_out) == ["search_knowledge_base", "lookup_order"]
    assert one_out["tool_choice"] == BODY["tool_choice"]
    # Removing the last tool has to take tool_choice with it as well.
    bare = without_tool({"tools": [{"name": "only"}], "tool_choice": "any"}, "only")
    assert "tools" not in bare and "tool_choice" not in bare


def test_the_deferral_picker_can_never_return_every_tool():
    rows = [{"name": "a", "tokens": 900}, {"name": "b", "tokens": 400},
            {"name": "c", "tokens": 100}]
    picked = defer_candidates(rows)
    assert picked == ["b", "c"]
    assert len(picked) < len(rows)
    # Naming every tool hot leaves nothing to defer, which is also fine.
    assert defer_candidates(rows, hot=["a", "b", "c"]) == []
    assert defer_candidates(rows, hot=["a"]) == ["b", "c"]
    assert defer_candidates([{"name": "only", "tokens": 10}]) == []
    assert defer_candidates([]) == []


def test_the_counting_body_keeps_what_is_being_measured():
    body = countable(BODY)
    assert "max_tokens" not in body and "temperature" not in body
    assert body["tools"] == BODY["tools"]
    assert body["model"] == "claude-opus-5"
    assert countable(None) == {}
    assert choice_kind(BODY) == "auto"
    assert choice_kind({"tool_choice": {"type": "tool", "name": "x"}}) == "any"
    assert choice_kind({"tool_choice": "any"}) == "any"
    assert choice_kind({}) == "auto"


def test_the_states_are_bounded_and_a_missing_number_stays_missing():
    assert classify(1000, 900)[0] == "schema-modest"
    assert classify(1000, 700)[0] == "schema-heavy"
    assert classify(1000, 500)[0] == "schema-dominates"
    assert classify(1000, 1000)[0] == "no-tools"
    assert classify(0, 0)[0] == "nothing-counted"
    assert overhead_share(0, 0) is None
    assert overhead(500, 900) == 0
    assert monthly_cost(11500, 0, 3.0) is None
    assert monthly_cost(11500, 10, "free") is None
    assert window_share(12388, 200000) == 0.06194
    assert window_share(12388, 0) is None

import { test } from 'node:test';
import assert from 'node:assert/strict';
import { choiceKind, classify, countable, deferCandidates, fixedOverhead,
         monthlyCost, overhead, overheadShare, systemPromptTokens, toolNames,
         windowShare, withoutTool, withoutTools }
  from './anthropic-tool-schema-overhead.mjs';

const BODY = {
  model: 'claude-opus-5',
  max_tokens: 1024,
  temperature: 0,
  system: 'You are a support agent.',
  tool_choice: { type: 'auto' },
  messages: [{ role: 'user', content: 'where is my order' }],
  tools: [
    { name: 'search_knowledge_base', input_schema: { type: 'object' } },
    { name: 'create_ticket', input_schema: { type: 'object' } },
    { name: 'lookup_order', input_schema: { type: 'object' } },
  ],
};

test('the tools block is most of what you pay for', () => {
  const total = 12388;
  const base = 888;
  assert.equal(overhead(total, base), 11500);
  assert.equal(Number(overheadShare(total, base).toFixed(4)), 0.9283);

  const [state, detail] = classify(total, base);
  assert.equal(state, 'schema-dominates');
  assert.match(detail, /11500 of 12388 input token/);
  assert.match(detail, /888 token\(s\) of system and messages, a ratio of 13.0 to 1/);

  assert.equal(monthlyCost(11500, 10000, 3.0), 10350);
});

test('the ablation deltas do not add up to the whole', () => {
  const perTool = [{ name: 'search_knowledge_base', tokens: 6200 },
                   { name: 'create_ticket', tokens: 3100 },
                   { name: 'lookup_order', tokens: 1914 }];
  const [residual, measured] = fixedOverhead(11500, perTool);
  assert.equal(measured, 11214);
  assert.equal(residual, 286);
  assert.equal(residual, systemPromptTokens('claude-opus-5', 'auto'));
  assert.deepEqual(fixedOverhead(0, perTool), [0, 11214]);
});

test('the system prompt table matches on longest prefix', () => {
  assert.equal(systemPromptTokens('claude-opus-5'), 286);
  assert.equal(systemPromptTokens('claude-opus-5', 'any'), 406);
  assert.equal(systemPromptTokens('claude-sonnet-5'), 354);
  assert.equal(systemPromptTokens('claude-opus-4-5'), 496);
  assert.equal(systemPromptTokens('claude-haiku-4-5-20251001'), 496);
  assert.equal(systemPromptTokens('claude-opus-4-7', 'any'), 804);
  assert.equal(systemPromptTokens('claude-fable-5'), null);
  assert.equal(systemPromptTokens(''), null);
  assert.equal(systemPromptTokens(null), null);
});

test('removing the tools removes the tool_choice with them', () => {
  const stripped = withoutTools(BODY);
  assert.equal('tools' in stripped, false);
  assert.equal('tool_choice' in stripped, false);
  assert.equal(stripped.system, BODY.system);
  assert.deepEqual(stripped.messages, BODY.messages);
  assert.equal(BODY.tools.length, 3);
  assert.equal('tool_choice' in BODY, true);

  const oneOut = withoutTool(BODY, 'create_ticket');
  assert.deepEqual(toolNames(oneOut), ['search_knowledge_base', 'lookup_order']);
  assert.deepEqual(oneOut.tool_choice, BODY.tool_choice);

  const bare = withoutTool({ tools: [{ name: 'only' }], tool_choice: 'any' }, 'only');
  assert.equal('tools' in bare, false);
  assert.equal('tool_choice' in bare, false);
});

test('the deferral picker can never return every tool', () => {
  const rows = [{ name: 'a', tokens: 900 }, { name: 'b', tokens: 400 },
                { name: 'c', tokens: 100 }];
  const picked = deferCandidates(rows);
  assert.deepEqual(picked, ['b', 'c']);
  assert.ok(picked.length < rows.length);
  assert.deepEqual(deferCandidates(rows, ['a', 'b', 'c']), []);
  assert.deepEqual(deferCandidates(rows, ['a']), ['b', 'c']);
  assert.deepEqual(deferCandidates([{ name: 'only', tokens: 10 }]), []);
  assert.deepEqual(deferCandidates([]), []);
});

test('the counting body keeps what is being measured', () => {
  const body = countable(BODY);
  assert.equal('max_tokens' in body, false);
  assert.equal('temperature' in body, false);
  assert.deepEqual(body.tools, BODY.tools);
  assert.equal(body.model, 'claude-opus-5');
  assert.deepEqual(countable(null), {});
  assert.equal(choiceKind(BODY), 'auto');
  assert.equal(choiceKind({ tool_choice: { type: 'tool', name: 'x' } }), 'any');
  assert.equal(choiceKind({ tool_choice: 'any' }), 'any');
  assert.equal(choiceKind({}), 'auto');
});

test('the states are bounded and a missing number stays missing', () => {
  assert.equal(classify(1000, 900)[0], 'schema-modest');
  assert.equal(classify(1000, 700)[0], 'schema-heavy');
  assert.equal(classify(1000, 500)[0], 'schema-dominates');
  assert.equal(classify(1000, 1000)[0], 'no-tools');
  assert.equal(classify(0, 0)[0], 'nothing-counted');
  assert.equal(overheadShare(0, 0), null);
  assert.equal(overhead(500, 900), 0);
  assert.equal(monthlyCost(11500, 0, 3.0), null);
  assert.equal(monthlyCost(11500, 10, 'free'), null);
  assert.equal(windowShare(12388, 200000), 0.06194);
  assert.equal(windowShare(12388, 0), null);
});