The same body counts 30% more tokens on the newer model

"""Measure the token delta between two Claude models on one identical body.

Claude 4.7 and later use a newer tokenizer that produces roughly 30 percent
more tokens for the same text, and the exact increase depends on the content.
This measures the increase for your bodies instead of assuming the headline.

The only non-GET request in this section: POST /v1/messages/count_tokens. It is
documented as free, it creates no message, it generates nothing, and its rate
limit is separate from message creation. Nothing else here contacts the API.

The two calls must differ only in the model field. A ratio taken across two
bodies that drifted apart measures the harness rather than the tokenizer, so
that is asserted before either request is sent.

This is a budgeting reading, not a ceiling one. It never asks whether a payload
fits: see prompt-too-long-context-overflow for max_input_tokens and
request-too-large-413 for the byte limit.
"""
import argparse
import json
import logging
import os
import sys

import requests

logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s")
log = logging.getLogger("anthropic_tokenizer_delta")

# POST to /v1/messages/count_tokens, and this is the only write-shaped call in
# the section. It creates nothing, returns no completion, and bills nothing.
COUNT_TOKENS_URL = "https://api.anthropic.com/v1/messages/count_tokens"

# Fields that belong to message creation and are not part of counting. Sending
# them is not useful and max_tokens in particular describes output, which the
# counting endpoint has no opinion about.
GENERATION_ONLY = ("max_tokens", "temperature", "top_p", "top_k", "stream",
                   "stop_sequences", "service_tier", "metadata")

# Below this the two tokenizers are the same one and the run is a no-op.
TOLERANCE = 0.02

MEASURED = "measured: two input_tokens values from count_tokens on identical bodies"
INFERRED = "inferred: that this ratio holds for traffic these %d bodies represent"

FINDINGS = ("tokenizer-delta", "count-failed", "bodies-differ")


def count_body(body):
    """A counting body from a Messages body. Pure. Generation fields removed.

    Everything that occupies the input window is kept: system, tools,
    tool_choice, thinking and the messages themselves, including images and
    documents. Only the knobs that describe generation are dropped.
    """
    if not isinstance(body, dict):
        return {}
    return {k: v for k, v in body.items() if k not in GENERATION_ONLY}


def swap_model(body, model):
    """The same body under a different model id. Pure. One key changes."""
    out = dict(body or {})
    out["model"] = str(model)
    return out


def same_apart_from_model(left, right):
    """True when the only difference is the model field. Pure.

    The guard the whole measurement rests on. Two counts of two different
    bodies is not a tokenizer ratio, it is noise with a decimal point.
    """
    a = {k: v for k, v in (left or {}).items() if k != "model"}
    b = {k: v for k, v in (right or {}).items() if k != "model"}
    return (json.dumps(a, sort_keys=True, separators=(",", ":"))
            == json.dumps(b, sort_keys=True, separators=(",", ":")))


def ratio(base, target):
    """target / base as a float. Pure. None when the base count is unusable."""
    try:
        base = int(base)
        target = int(target)
    except (TypeError, ValueError):
        return None
    if base <= 0:
        return None
    return target / base


def workload_ratio(rows):
    """Token-weighted ratio across the sample. Pure. None when nothing counted.

    Weighted, not averaged. A mean of per-body ratios lets a two-line fixture
    count as much as the 40k-token thread that is most of the bill.
    """
    base = sum(int(r.get("base_tokens") or 0) for r in rows or []
               if r.get("base_tokens"))
    target = sum(int(r.get("target_tokens") or 0) for r in rows or []
                 if r.get("target_tokens"))
    return ratio(base, target)


def rebaseline(budgets, r):
    """[(name, old, new)] for each declared constant. Pure. Sorted by name."""
    out = []
    if not r:
        return out
    for name in sorted(budgets or {}):
        old = int(budgets[name])
        out.append((name, old, int(round(old * r))))
    return out


def parse_budgets(raw):
    """{name: tokens} from name=tokens pairs. Pure. Bad pairs are dropped."""
    out = {}
    for item in raw or []:
        for part in str(item).split(","):
            if "=" not in part:
                continue
            name, _, value = part.partition("=")
            name = name.strip()
            try:
                tokens = int(str(value).strip().replace("_", ""))
            except ValueError:
                continue
            if name and tokens > 0:
                out[name] = tokens
    return out


def verdict(rows, base_model, target_model):
    """Grade the run. Pure. Returns (state, detail)."""
    rows = list(rows or [])
    if not rows:
        return ("no-bodies", "no bodies were counted, so there is nothing to "
                             "compare")
    failed = [r for r in rows if r.get("error")]
    if len(failed) == len(rows):
        return ("count-failed",
                "every count failed: %s" % failed[0].get("error"))
    if any(r.get("mismatch") for r in rows):
        return ("bodies-differ",
                "at least one pair of bodies differed by more than the model "
                "field, so no ratio was taken for it")
    r = workload_ratio(rows)
    if r is None:
        return ("count-failed", "no usable input_tokens came back")
    counted = [x for x in rows if not x.get("error")]
    if abs(r - 1.0) < TOLERANCE:
        return ("counts-agree",
                "%s and %s count this workload within %d%% of each other, so "
                "they share a tokenizer and no constant needs re-baselining"
                % (base_model, target_model, int(TOLERANCE * 100)))
    return ("tokenizer-delta",
            "the workload counts %.3fx more tokens on %s, measured over %d "
            "body/bodies" % (r, target_model, len(counted)))


def repair_lines(state, r):
    """The repair for one verdict. Pure. Printed, never performed."""
    if state == "tokenizer-delta":
        lines = ["re-baseline every constant above, and key any stored token "
                 "count by model as well as by text. A count with no model "
                 "attached is wrong for one of the two models and you cannot "
                 "tell which."]
        if r and r > 1:
            lines.append("expect input spend on this workload to move by about "
                         "%d%% at flat traffic, since billing follows the count "
                         "the model actually consumed." % round((r - 1) * 100))
            lines.append("prompts assembled to a fixed token budget now carry "
                         "less content than they did. Check retrieval quality "
                         "and any compaction threshold before blaming the model.")
        return lines
    if state == "bodies-differ":
        return ["the two bodies differed by more than the model field, so the "
                "ratio would have measured the harness. Count one body, swap "
                "only model, and send it twice."]
    if state == "count-failed":
        return ["read the error text above. A 400 naming the model is an id "
                "this account cannot reach; a 413 is the 32 MB byte ceiling, "
                "which is a different note."]
    if state == "counts-agree":
        return ["nothing to change here. Both ids are on the same tokenizer, "
                "so counts measured on one transfer to the other."]
    return []


def count_tokens(body, key, timeout=30):
    """One count. Returns (input_tokens, error). Free, and creates nothing."""
    headers = {"x-api-key": key,
               "anthropic-version": "2023-06-01",
               "content-type": "application/json"}
    try:
        # POST to /v1/messages/count_tokens: free, no completion, no billing.
        r = requests.post(COUNT_TOKENS_URL, headers=headers, json=body,
                          timeout=timeout)
    except requests.RequestException as exc:
        return (None, "request failed: %s" % exc)
    if r.status_code != 200:
        detail = ""
        try:
            detail = str((r.json().get("error") or {}).get("message") or "")
        except ValueError:
            detail = (r.text or "")[:160]
        return (None, "HTTP %d %s" % (r.status_code, detail))
    try:
        return (int(r.json()["input_tokens"]), None)
    except (ValueError, KeyError, TypeError):
        return (None, "no input_tokens in the response")


def main():
    ap = argparse.ArgumentParser(description=__doc__)
    ap.add_argument("--from", dest="base", required=True,
                    help="the model the counts were originally measured on")
    ap.add_argument("--to", dest="target", required=True,
                    help="the model you are migrating to")
    ap.add_argument("--body", action="append", default=[],
                    help="a JSON file holding one real Messages request body")
    ap.add_argument("--budget", action="append", default=[],
                    help="name=tokens, repeatable, for a constant in your code")
    args = ap.parse_args()

    key = os.environ.get("ANTHROPIC_API_KEY")
    if not key:
        log.error("set ANTHROPIC_API_KEY to a workspace key. It is used only "
                  "for POST /v1/messages/count_tokens, which is free and "
                  "creates nothing")
        return 2
    if not args.body:
        log.error("pass --body at least once with a real request body. A toy "
                  "message measures a toy ratio")
        return 2

    budgets = parse_budgets(list(args.budget)
                            + [os.environ.get("ANTHROPIC_TOKEN_BUDGETS", "")])
    rows = []
    for path in args.body:
        name = os.path.basename(path)
        try:
            with open(path, "r", encoding="utf-8") as fh:
                raw = json.load(fh)
        except (OSError, ValueError) as exc:
            rows.append({"name": name, "error": "unreadable: %s" % exc})
            log.warning("%-24s unreadable: %s", name, exc)
            continue

        base_body = swap_model(count_body(raw), args.base)
        target_body = swap_model(count_body(raw), args.target)
        if not same_apart_from_model(base_body, target_body):
            rows.append({"name": name, "mismatch": True})
            log.warning("%-24s the two bodies differ by more than model", name)
            continue

        base_tokens, base_err = count_tokens(base_body, key)
        target_tokens, target_err = count_tokens(target_body, key)
        err = base_err or target_err
        if err:
            rows.append({"name": name, "error": err})
            log.warning("%-24s %s", name, err)
            continue

        r = ratio(base_tokens, target_tokens)
        rows.append({"name": name, "base_tokens": base_tokens,
                     "target_tokens": target_tokens, "ratio": r})
        log.info("%-24s %s %7d -> %s %7d   x%.3f", name, args.base,
                 base_tokens, args.target, target_tokens, r or 0.0)

    state, detail = verdict(rows, args.base, args.target)
    r = workload_ratio(rows)
    emit = log.warning if state in FINDINGS else log.info
    emit("%-20s %s", state, detail)
    if state in ("tokenizer-delta", "counts-agree"):
        emit("  %s", MEASURED)
        emit("  %s", INFERRED % len([x for x in rows if x.get("ratio")]))
    for name, old, new in rebaseline(budgets, r):
        emit("  budget %-10s %9d -> %9d tokens of the old measurement",
             name, old, new)
    if not budgets:
        emit("  no budgets declared. Pass --budget name=tokens for each token "
             "constant in your code to see it re-baselined")
    for line in repair_lines(state, r):
        emit("  repair: %s", line)

    log.info("%d finding(s)", 1 if state in FINDINGS else 0)
    return 1 if state in FINDINGS else 0


if __name__ == "__main__":
    sys.exit(main())

/**
 * Measure the token delta between two Claude models on one identical body.
 *
 * Claude 4.7 and later use a newer tokenizer that produces roughly 30 percent
 * more tokens for the same text; the exact increase depends on the content.
 *
 * The only non-GET request in this section: POST /v1/messages/count_tokens,
 * which is free, creates no message and generates nothing.
 *
 * The two calls may differ only in the model field, which is asserted before
 * either one is sent. A budgeting reading, never a ceiling one.
 */
import { readFile } from 'node:fs/promises';
import path from 'node:path';

const COUNT_TOKENS_URL = 'https://api.anthropic.com/v1/messages/count_tokens';

const GENERATION_ONLY = new Set(['max_tokens', 'temperature', 'top_p', 'top_k',
  'stream', 'stop_sequences', 'service_tier', 'metadata']);

export const TOLERANCE = 0.02;

export const MEASURED =
  'measured: two input_tokens values from count_tokens on identical bodies';

const FINDINGS = new Set(['tokenizer-delta', 'count-failed', 'bodies-differ']);

/** A counting body from a Messages body. Pure. Generation fields removed. */
export function countBody(body) {
  if (!body || typeof body !== 'object' || Array.isArray(body)) return {};
  const out = {};
  for (const [k, v] of Object.entries(body)) {
    if (!GENERATION_ONLY.has(k)) out[k] = v;
  }
  return out;
}

/** The same body under a different model id. Pure. One key changes. */
export function swapModel(body, model) {
  return { ...(body ?? {}), model: String(model) };
}

const canonical = (value) => {
  if (Array.isArray(value)) return value.map(canonical);
  if (value && typeof value === 'object') {
    const out = {};
    for (const key of Object.keys(value).sort()) out[key] = canonical(value[key]);
    return out;
  }
  return value;
};

/** True when the only difference is the model field. Pure. */
export function sameApartFromModel(left, right) {
  const strip = (obj) => {
    const { model, ...rest } = obj ?? {};
    return JSON.stringify(canonical(rest));
  };
  return strip(left) === strip(right);
}

/** target / base. Pure. Null when the base count is unusable. */
export function ratio(base, target) {
  const b = Number(base);
  const t = Number(target);
  if (!Number.isFinite(b) || !Number.isFinite(t) || b <= 0) return null;
  return t / b;
}

/** Token-weighted ratio across the sample. Pure. Null when nothing counted. */
export function workloadRatio(rows) {
  let base = 0;
  let target = 0;
  for (const row of rows ?? []) {
    base += Number(row?.baseTokens ?? 0) || 0;
    target += Number(row?.targetTokens ?? 0) || 0;
  }
  return ratio(base, target);
}

/** [[name, old, new]] for each declared constant. Pure. Sorted by name. */
export function rebaseline(budgets, r) {
  if (!r) return [];
  return Object.keys(budgets ?? {}).sort()
    .map((name) => [name, Math.trunc(budgets[name]),
                    Math.round(Math.trunc(budgets[name]) * r)]);
}

/** {name: tokens} from name=tokens pairs. Pure. Bad pairs are dropped. */
export function parseBudgets(raw) {
  const out = {};
  for (const item of raw ?? []) {
    for (const part of String(item).split(',')) {
      const at = part.indexOf('=');
      if (at < 0) continue;
      const name = part.slice(0, at).trim();
      const tokens = Number.parseInt(part.slice(at + 1).trim().replace(/_/g, ''), 10);
      if (name && Number.isFinite(tokens) && tokens > 0) out[name] = tokens;
    }
  }
  return out;
}

/** Grade the run. Pure. Returns [state, detail]. */
export function verdict(rows, baseModel, targetModel) {
  const list = [...(rows ?? [])];
  if (!list.length) {
    return ['no-bodies', 'no bodies were counted, so there is nothing to compare'];
  }
  const failed = list.filter((r) => r?.error);
  if (failed.length === list.length) {
    return ['count-failed', `every count failed: ${failed[0].error}`];
  }
  if (list.some((r) => r?.mismatch)) {
    return ['bodies-differ',
      'at least one pair of bodies differed by more than the model field, so no '
      + 'ratio was taken for it'];
  }
  const r = workloadRatio(list);
  if (r === null) return ['count-failed', 'no usable input_tokens came back'];
  const counted = list.filter((x) => !x?.error);
  if (Math.abs(r - 1) < TOLERANCE) {
    return ['counts-agree',
      `${baseModel} and ${targetModel} count this workload within `
      + `${Math.trunc(TOLERANCE * 100)}% of each other, so they share a `
      + 'tokenizer and no constant needs re-baselining'];
  }
  return ['tokenizer-delta',
    `the workload counts ${r.toFixed(3)}x more tokens on ${targetModel}, `
    + `measured over ${counted.length} body/bodies`];
}

/** The repair for one verdict. Pure. Printed, never performed. */
export function repairLines(state, r) {
  if (state === 'tokenizer-delta') {
    const lines = ['re-baseline every constant above, and key any stored token '
      + 'count by model as well as by text. A count with no model attached is '
      + 'wrong for one of the two models and you cannot tell which.'];
    if (r && r > 1) {
      lines.push('expect input spend on this workload to move by about '
        + `${Math.round((r - 1) * 100)}% at flat traffic, since billing follows `
        + 'the count the model actually consumed.');
      lines.push('prompts assembled to a fixed token budget now carry less '
        + 'content than they did. Check retrieval quality and any compaction '
        + 'threshold before blaming the model.');
    }
    return lines;
  }
  if (state === 'bodies-differ') {
    return ['the two bodies differed by more than the model field, so the ratio '
      + 'would have measured the harness. Count one body, swap only model, and '
      + 'send it twice.'];
  }
  if (state === 'count-failed') {
    return ['read the error text above. A 400 naming the model is an id this '
      + 'account cannot reach; a 413 is the 32 MB byte ceiling, which is a '
      + 'different note.'];
  }
  if (state === 'counts-agree') {
    return ['nothing to change here. Both ids are on the same tokenizer, so '
      + 'counts measured on one transfer to the other.'];
  }
  return [];
}

async function countTokens(body, key) {
  let res;
  try {
    res = await fetch(COUNT_TOKENS_URL, {
      method: 'POST', // /v1/messages/count_tokens: free, creates and bills nothing
      headers: { 'x-api-key': key, 'anthropic-version': '2023-06-01',
                 'content-type': 'application/json' },
      body: JSON.stringify(body),
    });
  } catch (err) {
    return [null, `request failed: ${err.message}`];
  }
  if (res.status !== 200) {
    let detail = '';
    try { detail = String((await res.json())?.error?.message ?? ''); } catch { detail = ''; }
    return [null, `HTTP ${res.status} ${detail}`];
  }
  try {
    const parsed = await res.json();
    return [Math.trunc(Number(parsed.input_tokens)), null];
  } catch {
    return [null, 'no input_tokens in the response'];
  }
}

function args(argv) {
  const out = { body: [], budget: [] };
  for (let i = 0; i < argv.length; i += 1) {
    const flag = argv[i];
    if (flag === '--from') out.from = argv[i += 1];
    else if (flag === '--to') out.to = argv[i += 1];
    else if (flag === '--body') out.body.push(argv[i += 1]);
    else if (flag === '--budget') out.budget.push(argv[i += 1]);
  }
  return out;
}

async function main() {
  const opts = args(process.argv.slice(2));
  const key = process.env.ANTHROPIC_API_KEY;
  if (!key) {
    console.error('set ANTHROPIC_API_KEY to a workspace key. It is used only for '
      + 'POST /v1/messages/count_tokens, which is free and creates nothing');
    process.exitCode = 2;
    return;
  }
  if (!opts.from || !opts.to || !opts.body.length) {
    console.error('usage: --from <model> --to <model> --body <file.json> '
      + '[--budget name=tokens]');
    process.exitCode = 2;
    return;
  }

  const budgets = parseBudgets([...opts.budget,
                                process.env.ANTHROPIC_TOKEN_BUDGETS ?? '']);
  const rows = [];
  for (const file of opts.body) {
    const name = path.basename(file);
    let raw;
    try {
      raw = JSON.parse(await readFile(file, 'utf8'));
    } catch (err) {
      rows.push({ name, error: `unreadable: ${err.message}` });
      console.log(`${name.padEnd(24)} unreadable: ${err.message}`);
      continue;
    }
    const baseBody = swapModel(countBody(raw), opts.from);
    const targetBody = swapModel(countBody(raw), opts.to);
    if (!sameApartFromModel(baseBody, targetBody)) {
      rows.push({ name, mismatch: true });
      console.log(`${name.padEnd(24)} the two bodies differ by more than model`);
      continue;
    }
    const [baseTokens, baseErr] = await countTokens(baseBody, key);
    const [targetTokens, targetErr] = await countTokens(targetBody, key);
    const err = baseErr || targetErr;
    if (err) {
      rows.push({ name, error: err });
      console.log(`${name.padEnd(24)} ${err}`);
      continue;
    }
    const r = ratio(baseTokens, targetTokens);
    rows.push({ name, baseTokens, targetTokens, ratio: r });
    console.log(`${name.padEnd(24)} ${opts.from} ${baseTokens} -> ${opts.to} `
      + `${targetTokens}   x${(r ?? 0).toFixed(3)}`);
  }

  const [state, detail] = verdict(rows, opts.from, opts.to);
  const r = workloadRatio(rows);
  console.log(`${state.padEnd(20)} ${detail}`);
  if (state === 'tokenizer-delta' || state === 'counts-agree') {
    console.log(`  ${MEASURED}`);
    console.log(`  inferred: that this ratio holds for traffic these `
      + `${rows.filter((x) => x.ratio).length} bodies represent`);
  }
  for (const [name, old, next] of rebaseline(budgets, r)) {
    console.log(`  budget ${name.padEnd(10)} ${old} -> ${next} tokens of the old measurement`);
  }
  if (!Object.keys(budgets).length) {
    console.log('  no budgets declared. Pass --budget name=tokens for each token '
      + 'constant in your code to see it re-baselined');
  }
  for (const line of repairLines(state, r)) console.log(`  repair: ${line}`);
  console.log(`${FINDINGS.has(state) ? 1 : 0} finding(s)`);
  process.exitCode = FINDINGS.has(state) ? 1 : 0;
}

if (import.meta.url === `file://${process.argv[1]}`) await main();

from anthropic_tokenizer_delta import (TOLERANCE, count_body, parse_budgets,
                                       ratio, rebaseline, repair_lines,
                                       same_apart_from_model, swap_model,
                                       verdict, workload_ratio)

BODY = {
    "model": "claude-sonnet-4-6",
    "system": "You are a scientist",
    "messages": [{"role": "user", "content": "Hello, Claude"}],
    "tools": [{"name": "get_weather", "description": "weather",
               "input_schema": {"type": "object", "properties": {}}}],
    "thinking": {"type": "enabled", "budget_tokens": 16000},
    "max_tokens": 1024,
    "temperature": 0.2,
}


def test_a_body_that_drifted_never_produces_a_ratio():
    left = swap_model(count_body(BODY), "claude-sonnet-4-6")
    right = swap_model(count_body(BODY), "claude-opus-5")
    assert same_apart_from_model(left, right)
    # One word of drift in the system prompt looks exactly like a tokenizer
    # delta and is not one.
    drifted = dict(right, system="You are a careful scientist")
    assert not same_apart_from_model(left, drifted)
    state, detail = verdict([{"name": "a.json", "mismatch": True}],
                            "claude-sonnet-4-6", "claude-opus-5")
    assert state == "bodies-differ"
    assert "no ratio was taken" in detail
    assert any("swap only model" in line for line in repair_lines(state, None))


def test_the_workload_ratio_is_token_weighted_and_not_a_mean_of_ratios():
    rows = [{"base_tokens": 40000, "target_tokens": 52000, "ratio": 1.3},
            {"base_tokens": 200, "target_tokens": 400, "ratio": 2.0}]
    # A mean of the two ratios would be 1.65. The bill follows the tokens.
    assert abs(workload_ratio(rows) - (52400 / 40200)) < 1e-9
    assert workload_ratio(rows) < 1.32
    assert workload_ratio([]) is None
    assert workload_ratio([{"base_tokens": 0, "target_tokens": 10}]) is None


def test_two_ids_on_the_same_tokenizer_are_a_non_finding():
    rows = [{"name": "a.json", "base_tokens": 1000, "target_tokens": 1005,
             "ratio": 1.005}]
    state, detail = verdict(rows, "claude-opus-5", "claude-sonnet-5")
    assert state == "counts-agree"
    assert "share a tokenizer" in detail
    assert any("transfer to the other" in line for line in repair_lines(state, 1.005))
    assert abs(1.005 - 1.0) < TOLERANCE


def test_the_delta_is_reported_with_what_it_costs_and_what_it_breaks():
    rows = [{"name": "a.json", "base_tokens": 18204, "target_tokens": 23551,
             "ratio": 1.2937}]
    state, detail = verdict(rows, "claude-sonnet-4-6", "claude-opus-5")
    assert state == "tokenizer-delta"
    assert "claude-opus-5" in detail and "1.294" in detail
    lines = repair_lines(state, workload_ratio(rows))
    assert any("key any stored token count by model" in line for line in lines)
    assert any("29%" in line for line in lines)
    assert any("retrieval quality" in line for line in lines)


def test_counting_bodies_drop_generation_fields_and_keep_the_window():
    counted = count_body(BODY)
    assert "max_tokens" not in counted and "temperature" not in counted
    for kept in ("system", "messages", "tools", "thinking"):
        assert kept in counted
    assert count_body(None) == {}
    assert swap_model(counted, "claude-fable-5")["model"] == "claude-fable-5"
    # swap_model does not mutate what it was handed.
    assert counted["model"] == "claude-sonnet-4-6"


def test_budgets_are_parsed_forgivingly_and_rebaselined_in_order():
    budgets = parse_budgets(["history=120000,chunk=800", "junk", "bad=x",
                             "zero=0"])
    assert budgets == {"history": 120000, "chunk": 800}
    assert rebaseline(budgets, 1.33) == [("chunk", 800, 1064),
                                         ("history", 120000, 159600)]
    assert rebaseline(budgets, None) == []


def test_a_413_is_handed_to_the_byte_note_rather_than_counted():
    rows = [{"name": "big.json", "error": "HTTP 413 Request exceeds the "
                                          "maximum allowed number of bytes."}]
    state, detail = verdict(rows, "claude-sonnet-4-6", "claude-opus-5")
    assert state == "count-failed"
    assert "413" in detail
    assert any("32 MB byte ceiling" in line for line in repair_lines(state, None))
    assert ratio(0, 10) is None and ratio(None, 10) is None
    assert verdict([], "a", "b")[0] == "no-bodies"

import { test } from 'node:test';
import assert from 'node:assert/strict';
import { TOLERANCE, countBody, parseBudgets, ratio, rebaseline, repairLines,
         sameApartFromModel, swapModel, verdict,
         workloadRatio } from './anthropic-tokenizer-delta.mjs';

const BODY = {
  model: 'claude-sonnet-4-6',
  system: 'You are a scientist',
  messages: [{ role: 'user', content: 'Hello, Claude' }],
  tools: [{ name: 'get_weather', description: 'weather',
            input_schema: { type: 'object', properties: {} } }],
  thinking: { type: 'enabled', budget_tokens: 16000 },
  max_tokens: 1024,
  temperature: 0.2,
};

test('a body that drifted never produces a ratio', () => {
  const left = swapModel(countBody(BODY), 'claude-sonnet-4-6');
  const right = swapModel(countBody(BODY), 'claude-opus-5');
  assert.ok(sameApartFromModel(left, right));
  assert.ok(!sameApartFromModel(left, { ...right, system: 'You are a careful scientist' }));
  const [state, detail] = verdict([{ name: 'a.json', mismatch: true }],
                                  'claude-sonnet-4-6', 'claude-opus-5');
  assert.equal(state, 'bodies-differ');
  assert.ok(detail.includes('no ratio was taken'));
  assert.ok(repairLines(state, null).some((l) => l.includes('swap only model')));
});

test('the workload ratio is token weighted and not a mean of ratios', () => {
  const rows = [{ baseTokens: 40000, targetTokens: 52000, ratio: 1.3 },
                { baseTokens: 200, targetTokens: 400, ratio: 2.0 }];
  assert.ok(Math.abs(workloadRatio(rows) - (52400 / 40200)) < 1e-9);
  assert.ok(workloadRatio(rows) < 1.32);
  assert.equal(workloadRatio([]), null);
  assert.equal(workloadRatio([{ baseTokens: 0, targetTokens: 10 }]), null);
});

test('two ids on the same tokenizer are a non finding', () => {
  const rows = [{ name: 'a.json', baseTokens: 1000, targetTokens: 1005, ratio: 1.005 }];
  const [state, detail] = verdict(rows, 'claude-opus-5', 'claude-sonnet-5');
  assert.equal(state, 'counts-agree');
  assert.ok(detail.includes('share a tokenizer'));
  assert.ok(repairLines(state, 1.005).some((l) => l.includes('transfer to the other')));
  assert.ok(Math.abs(1.005 - 1) < TOLERANCE);
});

test('the delta is reported with what it costs and what it breaks', () => {
  const rows = [{ name: 'a.json', baseTokens: 18204, targetTokens: 23551, ratio: 1.2937 }];
  const [state, detail] = verdict(rows, 'claude-sonnet-4-6', 'claude-opus-5');
  assert.equal(state, 'tokenizer-delta');
  assert.ok(detail.includes('claude-opus-5') && detail.includes('1.294'));
  const lines = repairLines(state, workloadRatio(rows));
  assert.ok(lines.some((l) => l.includes('key any stored token count by model')));
  assert.ok(lines.some((l) => l.includes('29%')));
  assert.ok(lines.some((l) => l.includes('retrieval quality')));
});

test('counting bodies drop generation fields and keep the window', () => {
  const counted = countBody(BODY);
  assert.ok(!('max_tokens' in counted) && !('temperature' in counted));
  for (const kept of ['system', 'messages', 'tools', 'thinking']) {
    assert.ok(kept in counted);
  }
  assert.deepEqual(countBody(null), {});
  assert.equal(swapModel(counted, 'claude-fable-5').model, 'claude-fable-5');
  assert.equal(counted.model, 'claude-sonnet-4-6');
});

test('budgets are parsed forgivingly and rebaselined in order', () => {
  const budgets = parseBudgets(['history=120000,chunk=800', 'junk', 'bad=x', 'zero=0']);
  assert.deepEqual(budgets, { history: 120000, chunk: 800 });
  assert.deepEqual(rebaseline(budgets, 1.33),
                   [['chunk', 800, 1064], ['history', 120000, 159600]]);
  assert.deepEqual(rebaseline(budgets, null), []);
});

test('a 413 is handed to the byte note rather than counted', () => {
  const rows = [{ name: 'big.json',
                  error: 'HTTP 413 Request exceeds the maximum allowed number of bytes.' }];
  const [state, detail] = verdict(rows, 'claude-sonnet-4-6', 'claude-opus-5');
  assert.equal(state, 'count-failed');
  assert.ok(detail.includes('413'));
  assert.ok(repairLines(state, null).some((l) => l.includes('32 MB byte ceiling')));
  assert.equal(ratio(0, 10), null);
  assert.equal(ratio(null, 10), null);
  assert.equal(verdict([], 'a', 'b')[0], 'no-bodies');
});