Prompts overflow the context window and 400 as too long

"""Pre-flight a Claude payload against the model's context window.

Read only, with one deliberate exception. Nothing here creates a completion:
the payload goes to /v1/messages/count_tokens, which is free, generates no
output, creates no object and bills nothing. It returns an input_tokens number
and runs against its own rate limit. That is the only way to learn what a body
costs in tokens without paying for an answer, so it is the one non-GET call in
this script. Everything else is a GET, and /v1/messages is never called.

The repair is printed, never applied. Deciding which half of a conversation to
drop is a product decision, not the side effect of an audit.
"""
import argparse
import json
import logging
import os
import sys

import requests

logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s")
log = logging.getLogger("anthropic_context_preflight")

API = "https://api.anthropic.com/v1"
VERSION = "2023-06-01"

# count_tokens takes the same structured body as message creation minus the
# parameters that only mean something when text is actually generated. Sending
# max_tokens to it is a 400, which is a confusing way for a pre-flight to fail,
# so these are stripped by name rather than hoped over.
SAMPLING_ONLY = ("max_tokens", "stream", "temperature", "top_p", "top_k",
                 "stop_sequences", "metadata", "service_tier")

OVERFLOW_STOP = "model_context_window_exceeded"
TOO_LONG = "prompt is too long"

FINDINGS = ("input-over-window", "budget-over-window", "window-tight")


def count_body(body):
    """The subset of a Messages body the counting endpoint accepts. Pure.

    Everything structural stays: system, messages, tools, tool_choice, thinking.
    All of it occupies the window, so dropping any of it to make the count
    simpler would produce a number about a request you are not sending.
    """
    if not isinstance(body, dict):
        return {}
    return {k: v for k, v in body.items() if k not in SAMPLING_ONLY}


def window_of(model_obj):
    """max_input_tokens off a model object, or None. Pure.

    None is not a large window. The field is returned by the API, but a proxy
    or gateway that reshapes the model object can drop it, and a ceiling that
    went missing has to stay missing rather than defaulting to something
    generous enough to let every payload pass.
    """
    if not isinstance(model_obj, dict):
        return None
    value = model_obj.get("max_input_tokens")
    return value if isinstance(value, int) and value > 0 else None


def budget(counted_input, max_tokens):
    """What one request reserves in the window. Pure.

    Input plus the room set aside for output, because max_tokens occupies the
    window whether or not the model uses it. Checking input alone is the common
    version of this check and it passes requests that are going to fail.
    """
    return int(counted_input or 0) + max(0, int(max_tokens or 0))


def verdict(counted_input, max_tokens, window, tight=0.9):
    """Classify one payload against one model's window. Pure. (state, detail).

    Two overflow states rather than one, because they do not fail alike: over
    on input alone is a 400 on every model, and over on the reservation is a
    200 on Claude 4.5 and newer that stops with model_context_window_exceeded.
    """
    counted_input = int(counted_input or 0)
    reserved = budget(counted_input, max_tokens)

    if window is None:
        return ("window-unknown",
                "%d input token(s) counted, and the model object carried no "
                "max_input_tokens, so there is no ceiling to compare against"
                % counted_input)

    shape = ("%d input + %d max_tokens = %d of a %d token window"
             % (counted_input, max(0, int(max_tokens or 0)), reserved, window))

    if counted_input > window:
        return ("input-over-window",
                "%s. The input alone is over the window, so this 400s with "
                "prompt is too long on every model, before max_tokens is even "
                "considered." % shape)
    if reserved > window:
        return ("budget-over-window",
                "%s. The input fits and the reservation does not. On Claude 4.5 "
                "and newer that returns 200 with stop_reason %s, which a client "
                "checking only for end_turn files as a complete answer."
                % (shape, OVERFLOW_STOP))

    share = reserved / float(window)
    if share >= tight:
        return ("window-tight",
                "%s (%.0f%%). It fits today and one longer turn ends that."
                % (shape, share * 100))
    return ("fits", "%s (%.0f%%)." % (shape, share * 100))


def turns_remaining(counted_input, max_tokens, window, per_turn):
    """How many more turns of `per_turn` tokens fit. Pure. None if unanswerable.

    A conversational product's real question is not whether this payload fits
    but how many exchanges are left before one stops fitting, and that is the
    number that turns an overflow into a scheduled piece of work.
    """
    if not window or not per_turn or per_turn <= 0:
        return None
    room = window - budget(counted_input, max_tokens)
    return max(0, int(room // per_turn))


def batch_overflows(lines):
    """Find window overflows in a batch results stream. Pure.

    Both shapes, because the same fault wears two faces. A succeeded result
    carrying stop_reason model_context_window_exceeded is the 200 nobody
    noticed; an errored result whose message says the prompt is too long is the
    400. Keyed by custom_id and never by position: results arrive in any order.
    """
    out = {}
    for line in lines or []:
        record = line
        if isinstance(record, (str, bytes)):
            text = record.decode("utf-8") if isinstance(record, bytes) else record
            text = text.strip()
            if not text:
                continue
            try:
                record = json.loads(text)
            except ValueError:
                continue
        if not isinstance(record, dict):
            continue

        custom_id = record.get("custom_id")
        result = record.get("result") or {}
        message = result.get("message") or {}
        if message.get("stop_reason") == OVERFLOW_STOP:
            out[custom_id] = "truncated-with-200"
            continue
        error = result.get("error") or {}
        if TOO_LONG in str(error.get("message") or "").lower():
            out[custom_id] = "rejected-with-400"
    return out


def get(session, path):
    """Every model and batch read in this script. GET only."""
    r = session.get(API + path, timeout=30)
    if r.status_code in (401, 403):
        raise SystemExit("%d from Anthropic: ANTHROPIC_API_KEY has to be a "
                         "workspace key that can reach /v1/models" % r.status_code)
    r.raise_for_status()
    return r.json()


def count_tokens(session, body):
    """The one call here that is not a GET, and it is not a write either.

    /v1/messages/count_tokens creates no object, generates no completion and
    is not billed. It carries its own rate limit, so a pre-flight on every
    request does not eat into the message limiter. A 413 back from it means the
    body is over the 32 MB byte ceiling, which is a different problem with a
    different note.
    """
    r = session.post(API + "/messages/count_tokens",
                     json=count_body(body), timeout=60)
    if r.status_code == 413:
        raise SystemExit("413 from the counting endpoint: this body is over the "
                         "32 MB request ceiling, which is a byte problem rather "
                         "than a token one")
    r.raise_for_status()
    return int((r.json() or {}).get("input_tokens") or 0)


def batch_results(session, batch_id):
    """Stream one batch's results file. GET, and read as lines."""
    r = session.get(API + "/messages/batches/" + str(batch_id) + "/results",
                    timeout=120, stream=True)
    r.raise_for_status()
    return list(r.iter_lines(decode_unicode=True))


def main():
    ap = argparse.ArgumentParser(description=__doc__)
    ap.add_argument("--payload", action="append", default=[], metavar="FILE",
                    help="a JSON file holding a real Messages request body")
    ap.add_argument("--batch-id", action="append", default=[],
                    help="also scan a finished batch's results for overflows")
    ap.add_argument("--per-turn", type=int, default=0,
                    help="average tokens one conversational turn adds, used to "
                         "report how many turns of headroom are left")
    ap.add_argument("--tight", type=float, default=0.9,
                    help="share of the window above which a payload that still "
                         "fits is reported anyway (default 0.9)")
    ap.add_argument("--show-all", action="store_true",
                    help="also print payloads with plenty of window left")
    args = ap.parse_args()

    key = os.environ.get("ANTHROPIC_API_KEY")
    if not key:
        log.error("set ANTHROPIC_API_KEY to a workspace key")
        return 2
    if not args.payload and not args.batch_id:
        log.error("give at least one --payload FILE or --batch-id ID")
        return 2

    session = requests.Session()
    session.headers.update({"x-api-key": key, "anthropic-version": VERSION,
                            "content-type": "application/json"})

    windows = {}
    checked = 0
    bad = 0

    for path in args.payload:
        with open(path, "r", encoding="utf-8") as fh:
            body = json.load(fh)
        model = str(body.get("model") or "")
        if not model:
            bad += 1
            log.warning("%-20s %-30s no model field, so there is no window to "
                        "check it against", "no-model", path)
            continue
        if model not in windows:
            windows[model] = window_of(get(session, "/models/" + model))

        counted = count_tokens(session, body)
        state, detail = verdict(counted, body.get("max_tokens"),
                                windows[model], args.tight)
        checked += 1
        line = "%-20s %-30s %s" % (state, path, detail)
        if state in FINDINGS or state == "window-unknown":
            if state in FINDINGS:
                bad += 1
            log.warning(line)
        elif args.show_all:
            log.info(line)

        left = turns_remaining(counted, body.get("max_tokens"),
                               windows[model], args.per_turn)
        if left is not None:
            log.info("  room for %d more turn(s) at %d tokens each",
                     left, args.per_turn)
        if state in FINDINGS:
            log.warning("  repair: server side compaction (compact-2026-01-12) "
                        "for long conversations, context editing "
                        "(clear_tool_uses_20250919 / clear_thinking_20251015) "
                        "for agent loops, or the tool search tool so tool "
                        "definitions stop being resident on every turn")
            log.warning("  repair: caching does not help here. Cached tokens "
                        "still occupy the window; they only cost less.")

    for batch_id in args.batch_id:
        found = batch_overflows(batch_results(session, batch_id))
        checked += len(found)
        for custom_id, shape in sorted(found.items(), key=lambda kv: str(kv[0])):
            bad += 1
            log.warning("%-20s %-30s in batch %s", shape, custom_id, batch_id)

    log.info("%d payload(s) and batch result(s) checked, %d finding(s)",
             checked, bad)
    return 1 if bad else 0


if __name__ == "__main__":
    sys.exit(main())

/**
 * Pre-flight a Claude payload against the model's context window.
 *
 * Read only, with one deliberate exception. Nothing here creates a completion:
 * the payload goes to /v1/messages/count_tokens, which is free, generates no
 * output, creates no object and bills nothing. Everything else is a GET, and
 * /v1/messages is never called.
 *
 * The repair is printed, never applied.
 */
import { readFile } from 'node:fs/promises';

const API = 'https://api.anthropic.com/v1';
const VERSION = '2023-06-01';

const SAMPLING_ONLY = new Set(['max_tokens', 'stream', 'temperature', 'top_p',
  'top_k', 'stop_sequences', 'metadata', 'service_tier']);

const OVERFLOW_STOP = 'model_context_window_exceeded';
const TOO_LONG = 'prompt is too long';

const FINDINGS = new Set(['input-over-window', 'budget-over-window', 'window-tight']);

/** The subset of a Messages body the counting endpoint accepts. Pure. */
export function countBody(body) {
  if (!body || typeof body !== 'object') return {};
  return Object.fromEntries(
    Object.entries(body).filter(([k]) => !SAMPLING_ONLY.has(k)));
}

/**
 * max_input_tokens off a model object, or null. Pure.
 * Null is not a large window: a ceiling a gateway dropped has to stay missing
 * rather than defaulting to something every payload fits under.
 */
export function windowOf(modelObj) {
  if (!modelObj || typeof modelObj !== 'object') return null;
  const value = modelObj.max_input_tokens;
  return Number.isInteger(value) && value > 0 ? value : null;
}

/** What one request reserves in the window: input plus room for output. Pure. */
export function budget(countedInput, maxTokens) {
  return Math.trunc(countedInput || 0) + Math.max(0, Math.trunc(maxTokens || 0));
}

/** Classify one payload against one model's window. Pure. [state, detail]. */
export function verdict(countedInput, maxTokens, window, tight = 0.9) {
  const input = Math.trunc(countedInput || 0);
  const reserved = budget(input, maxTokens);

  if (window === null || window === undefined) {
    return ['window-unknown',
      `${input} input token(s) counted, and the model object carried no ` +
      'max_input_tokens, so there is no ceiling to compare against'];
  }

  const room = Math.max(0, Math.trunc(maxTokens || 0));
  const shape = `${input} input + ${room} max_tokens = ${reserved} of a ` +
                `${window} token window`;

  if (input > window) {
    return ['input-over-window',
      `${shape}. The input alone is over the window, so this 400s with prompt ` +
      'is too long on every model, before max_tokens is even considered.'];
  }
  if (reserved > window) {
    return ['budget-over-window',
      `${shape}. The input fits and the reservation does not. On Claude 4.5 ` +
      `and newer that returns 200 with stop_reason ${OVERFLOW_STOP}, which a ` +
      'client checking only for end_turn files as a complete answer.'];
  }

  const share = reserved / window;
  const pct = (share * 100).toFixed(0);
  if (share >= tight) {
    return ['window-tight',
      `${shape} (${pct}%). It fits today and one longer turn ends that.`];
  }
  return ['fits', `${shape} (${pct}%).`];
}

/** How many more turns of `perTurn` tokens fit. Pure. null if unanswerable. */
export function turnsRemaining(countedInput, maxTokens, window, perTurn) {
  if (!window || !perTurn || perTurn <= 0) return null;
  const room = window - budget(countedInput, maxTokens);
  return Math.max(0, Math.floor(room / perTurn));
}

/**
 * Find window overflows in a batch results stream. Pure.
 * Both shapes: a 200 carrying the overflow stop reason, and an errored result
 * whose message says the prompt is too long. Keyed by custom_id, never by
 * position, because results arrive in any order.
 */
export function batchOverflows(lines) {
  const out = {};
  for (const line of lines ?? []) {
    let record = line;
    if (typeof record === 'string') {
      const text = record.trim();
      if (!text) continue;
      try { record = JSON.parse(text); } catch { continue; }
    }
    if (!record || typeof record !== 'object') continue;

    const customId = record.custom_id;
    const result = record.result ?? {};
    const message = result.message ?? {};
    if (message.stop_reason === OVERFLOW_STOP) {
      out[customId] = 'truncated-with-200';
      continue;
    }
    const error = result.error ?? {};
    if (String(error.message ?? '').toLowerCase().includes(TOO_LONG)) {
      out[customId] = 'rejected-with-400';
    }
  }
  return out;
}

function headers(key) {
  return { 'x-api-key': key, 'anthropic-version': VERSION,
           'content-type': 'application/json' };
}

async function get(key, path) {
  const res = await fetch(API + path, { headers: headers(key) });
  if (res.status === 401 || res.status === 403) {
    throw new Error(`${res.status} from Anthropic: ANTHROPIC_API_KEY has to be ` +
                    'a workspace key that can reach /v1/models');
  }
  if (!res.ok) throw new Error(`${res.status} from ${path}`);
  return res.json();
}

/**
 * The one call here that is not a GET, and not a write either: the counting
 * endpoint creates nothing, generates nothing and is not billed.
 */
async function countTokens(key, body) {
  const res = await fetch(`${API}/messages/count_tokens`, {
    method: 'POST',  // count_tokens creates nothing and bills nothing
    headers: headers(key),
    body: JSON.stringify(countBody(body)),
  });
  if (res.status === 413) {
    throw new Error('413 from the counting endpoint: this body is over the ' +
                    '32 MB request ceiling, which is a byte problem rather ' +
                    'than a token one');
  }
  if (!res.ok) throw new Error(`${res.status} from /messages/count_tokens`);
  return Math.trunc((await res.json())?.input_tokens ?? 0);
}

async function batchResults(key, batchId) {
  const res = await fetch(`${API}/messages/batches/${batchId}/results`,
                          { headers: headers(key) });
  if (!res.ok) throw new Error(`${res.status} from batch ${batchId} results`);
  return (await res.text()).split('\n');
}

async function main() {
  const key = process.env.ANTHROPIC_API_KEY;
  if (!key) {
    console.error('set ANTHROPIC_API_KEY to a workspace key');
    process.exitCode = 2;
    return;
  }
  const paths = process.argv.slice(2).filter((a) => !a.startsWith('--'));
  const batchIds = (process.env.BATCH_IDS ?? '').split(',')
    .map((s) => s.trim()).filter(Boolean);
  if (paths.length === 0 && batchIds.length === 0) {
    console.error('pass one or more payload JSON files, or set BATCH_IDS');
    process.exitCode = 2;
    return;
  }
  const perTurn = Math.trunc(Number(process.env.PER_TURN ?? 0));
  const tight = Number(process.env.TIGHT ?? 0.9);
  const showAll = process.env.SHOW_ALL === '1';

  const windows = new Map();
  let checked = 0;
  let bad = 0;

  for (const path of paths) {
    const body = JSON.parse(await readFile(path, 'utf8'));
    const model = String(body.model ?? '');
    if (!model) {
      bad += 1;
      console.warn(`${'no-model'.padEnd(20)} ${path.padEnd(30)} no model field, ` +
                   'so there is no window to check it against');
      continue;
    }
    if (!windows.has(model)) windows.set(model, windowOf(await get(key, `/models/${model}`)));

    const counted = await countTokens(key, body);
    const [state, detail] = verdict(counted, body.max_tokens, windows.get(model), tight);
    checked += 1;
    const line = `${state.padEnd(20)} ${path.padEnd(30)} ${detail}`;
    if (FINDINGS.has(state) || state === 'window-unknown') {
      if (FINDINGS.has(state)) bad += 1;
      console.warn(line);
    } else if (showAll) {
      console.log(line);
    }

    const left = turnsRemaining(counted, body.max_tokens, windows.get(model), perTurn);
    if (left !== null) console.log(`  room for ${left} more turn(s) at ${perTurn} tokens each`);
    if (FINDINGS.has(state)) {
      console.warn('  repair: server side compaction (compact-2026-01-12) for long ' +
                   'conversations, context editing (clear_tool_uses_20250919 / ' +
                   'clear_thinking_20251015) for agent loops, or the tool search ' +
                   'tool so tool definitions stop being resident on every turn');
      console.warn('  repair: caching does not help here. Cached tokens still ' +
                   'occupy the window; they only cost less.');
    }
  }

  for (const batchId of batchIds) {
    const found = batchOverflows(await batchResults(key, batchId));
    const ids = Object.keys(found).sort();
    checked += ids.length;
    for (const customId of ids) {
      bad += 1;
      console.warn(`${found[customId].padEnd(20)} ${String(customId).padEnd(30)} ` +
                   `in batch ${batchId}`);
    }
  }

  console.log(`${checked} payload(s) and batch result(s) checked, ${bad} finding(s)`);
  process.exitCode = bad ? 1 : 0;
}

if (import.meta.url === `file://${process.argv[1]}`) {
  main().catch((err) => { console.error(err.message); process.exitCode = 2; });
}

from anthropic_context_preflight import (batch_overflows, budget, count_body,
                                          turns_remaining, verdict, window_of)


def test_input_fits_but_the_reservation_does_not():
    # The whole note in two assertions. 190k of input under a 200k window is
    # fine; the same input with a routine max_tokens is over, and it is over in
    # the way that comes back as a 200 rather than as a 400.
    ok_state, _ = verdict(190_000, 0, 200_000)
    assert ok_state == "window-tight"

    state, detail = verdict(190_000, 16_000, 200_000)
    assert state == "budget-over-window"
    assert "190000 input + 16000 max_tokens = 206000 of a 200000 token window" in detail
    assert "model_context_window_exceeded" in detail
    assert "200" in detail


def test_input_alone_over_the_window_is_the_other_failure():
    state, detail = verdict(260_000, 4_000, 200_000)
    assert state == "input-over-window"
    assert "prompt is too long" in detail
    assert budget(260_000, 4_000) == 264_000


def test_a_comfortable_payload_is_not_a_finding():
    state, detail = verdict(40_000, 8_000, 200_000)
    assert state == "fits"
    assert "(24%)" in detail


def test_the_counting_endpoint_only_gets_the_keys_it_accepts():
    body = {"model": "claude-sonnet-5", "system": "s", "messages": [],
            "tools": [{"name": "t"}], "tool_choice": {"type": "auto"},
            "thinking": {"type": "enabled"}, "max_tokens": 16_000,
            "temperature": 0.2, "stream": True, "service_tier": "auto"}
    trimmed = count_body(body)
    # Sampling parameters out, because count_tokens 400s on them.
    assert "max_tokens" not in trimmed
    assert "temperature" not in trimmed
    assert "stream" not in trimmed
    assert "service_tier" not in trimmed
    # Everything that occupies the window stays, because dropping any of it
    # would count a request you are not sending.
    assert set(trimmed) == {"model", "system", "messages", "tools",
                            "tool_choice", "thinking"}
    assert count_body(None) == {}


def test_a_missing_window_is_not_an_infinite_one():
    assert window_of({"id": "claude-sonnet-5", "max_input_tokens": 200_000}) == 200_000
    assert window_of({"id": "claude-sonnet-5"}) is None
    assert window_of({"max_input_tokens": 0}) is None
    assert window_of({"max_input_tokens": "200000"}) is None
    assert window_of(None) is None
    state, detail = verdict(500_000, 8_000, None)
    assert state == "window-unknown"
    assert "no max_input_tokens" in detail


def test_turns_remaining_is_the_number_a_product_team_wants():
    assert turns_remaining(120_000, 16_000, 200_000, 1_800) == 35
    assert turns_remaining(199_000, 16_000, 200_000, 1_800) == 0
    assert turns_remaining(120_000, 16_000, None, 1_800) is None
    assert turns_remaining(120_000, 16_000, 200_000, 0) is None


def test_batch_results_yield_both_shapes_keyed_by_custom_id():
    lines = [
        '{"custom_id": "doc-9", "result": {"type": "succeeded", "message": '
        '{"stop_reason": "model_context_window_exceeded"}}}',
        '{"custom_id": "doc-3", "result": {"type": "errored", "error": '
        '{"type": "invalid_request_error", "message": "prompt is too long: '
        '412000 tokens > 200000 maximum"}}}',
        '{"custom_id": "doc-1", "result": {"type": "succeeded", "message": '
        '{"stop_reason": "end_turn"}}}',
        "",
        "not json at all",
    ]
    assert batch_overflows(lines) == {"doc-9": "truncated-with-200",
                                      "doc-3": "rejected-with-400"}
    assert batch_overflows([]) == {}
    assert batch_overflows(None) == {}

import { test } from 'node:test';
import assert from 'node:assert/strict';
import { batchOverflows, budget, countBody, turnsRemaining, verdict, windowOf }
  from './anthropic-context-preflight.mjs';

test('input fits but the reservation does not', () => {
  assert.equal(verdict(190000, 0, 200000)[0], 'window-tight');
  const [state, detail] = verdict(190000, 16000, 200000);
  assert.equal(state, 'budget-over-window');
  assert.match(detail, /190000 input \+ 16000 max_tokens = 206000 of a 200000 token window/);
  assert.match(detail, /model_context_window_exceeded/);
  assert.match(detail, /200/);
});

test('input alone over the window is the other failure', () => {
  const [state, detail] = verdict(260000, 4000, 200000);
  assert.equal(state, 'input-over-window');
  assert.match(detail, /prompt is too long/);
  assert.equal(budget(260000, 4000), 264000);
});

test('a comfortable payload is not a finding', () => {
  const [state, detail] = verdict(40000, 8000, 200000);
  assert.equal(state, 'fits');
  assert.match(detail, /\(24%\)/);
});

test('the counting endpoint only gets the keys it accepts', () => {
  const trimmed = countBody({
    model: 'claude-sonnet-5', system: 's', messages: [], tools: [{ name: 't' }],
    tool_choice: { type: 'auto' }, thinking: { type: 'enabled' },
    max_tokens: 16000, temperature: 0.2, stream: true, service_tier: 'auto',
  });
  assert.deepEqual(Object.keys(trimmed).sort(),
    ['messages', 'model', 'system', 'thinking', 'tool_choice', 'tools']);
  assert.deepEqual(countBody(null), {});
});

test('a missing window is not an infinite one', () => {
  assert.equal(windowOf({ id: 'claude-sonnet-5', max_input_tokens: 200000 }), 200000);
  assert.equal(windowOf({ id: 'claude-sonnet-5' }), null);
  assert.equal(windowOf({ max_input_tokens: 0 }), null);
  assert.equal(windowOf({ max_input_tokens: '200000' }), null);
  assert.equal(windowOf(null), null);
  const [state, detail] = verdict(500000, 8000, null);
  assert.equal(state, 'window-unknown');
  assert.match(detail, /no max_input_tokens/);
});

test('turnsRemaining is the number a product team wants', () => {
  assert.equal(turnsRemaining(120000, 16000, 200000, 1800), 35);
  assert.equal(turnsRemaining(199000, 16000, 200000, 1800), 0);
  assert.equal(turnsRemaining(120000, 16000, null, 1800), null);
  assert.equal(turnsRemaining(120000, 16000, 200000, 0), null);
});

test('batch results yield both shapes keyed by custom_id', () => {
  const lines = [
    '{"custom_id": "doc-9", "result": {"type": "succeeded", "message": {"stop_reason": "model_context_window_exceeded"}}}',
    '{"custom_id": "doc-3", "result": {"type": "errored", "error": {"type": "invalid_request_error", "message": "prompt is too long: 412000 tokens > 200000 maximum"}}}',
    '{"custom_id": "doc-1", "result": {"type": "succeeded", "message": {"stop_reason": "end_turn"}}}',
    '',
    'not json at all',
  ];
  assert.deepEqual(batchOverflows(lines),
    { 'doc-9': 'truncated-with-200', 'doc-3': 'rejected-with-400' });
  assert.deepEqual(batchOverflows([]), {});
  assert.deepEqual(batchOverflows(null), {});
});