streamed responses report no usage and the dashboard undercounts

"""Reconcile OpenAI's token totals against the ones your own telemetry recorded.

Read only. Two GET requests against the organization endpoints and a JSON file
you supply. Those endpoints reject project keys, so this needs an organization
admin key (sk-admin-), which can and should be provisioned read-only.

The finding is a gap between two sources, not a problem with either provider's
billing. Streamed responses carry usage: null on every chunk unless the request
asked for the totals, so a dashboard built on per-request telemetry undercounts
by whatever share of the traffic streams. The repair is printed, not applied.
"""
import argparse
import json
import logging
import os
import sys
import time

import requests

logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s")
log = logging.getLogger("openai_streaming_usage_gap")

API = "https://api.openai.com/v1"

FINDINGS = ("undercount", "overcount", "untracked", "phantom")


def api_totals(buckets):
    """Fold usage buckets into one row per project. Pure.

    Requests are carried alongside the tokens because OpenAI reports them and
    Anthropic does not; a project with requests and no output tokens is a
    different note, and this one at least keeps the number in view.
    """
    rows = {}
    for bucket in buckets or []:
        for result in bucket.get("results") or []:
            project = str(result.get("project_id") or "unknown")
            row = rows.setdefault(project, {"tokens": 0, "requests": 0})
            row["tokens"] += (int(result.get("input_tokens") or 0)
                              + int(result.get("output_tokens") or 0))
            row["requests"] += int(result.get("num_model_requests") or 0)
    return rows


def recorded_tokens(entry):
    """Read one project's own recorded token count. Pure.

    Returns an int, or None when nothing was recorded for that project at all.
    The distinction is the point: zero means your pipeline saw the project and
    recorded nothing, None means it has never heard of it, and those are two
    different bugs with two different owners.
    """
    if entry is None:
        return None
    if isinstance(entry, bool):
        return None
    if isinstance(entry, (int, float)):
        return int(entry)
    if isinstance(entry, dict):
        if "tokens" in entry:
            try:
                return int(entry["tokens"] or 0)
            except (TypeError, ValueError):
                return None
        if "input_tokens" in entry or "output_tokens" in entry:
            try:
                return (int(entry.get("input_tokens") or 0)
                        + int(entry.get("output_tokens") or 0))
            except (TypeError, ValueError):
                return None
    return None


def compare(api_tokens, recorded, tolerance=0.05, min_tokens=100000):
    """Compare one project's two numbers. Pure. Returns (state, detail).

    Three disagreements, not one. Recorded below the API is the undercount this
    note is about. Recorded above it is double counting, a different bug that
    would be hidden by an absolute-value comparison. A project missing from the
    telemetry entirely is not undercounted, it is unrecorded.
    """
    api_tokens = int(api_tokens or 0)

    if api_tokens <= 0:
        if recorded is None or int(recorded) <= 0:
            return ("idle", "no usage in the org report and none recorded")
        return ("phantom",
                "%d token(s) recorded against a project the org report shows no "
                "usage for. That is a project id mapping, not a streaming "
                "problem." % int(recorded))

    if recorded is None:
        return ("untracked",
                "%d token(s) in the org report and no telemetry for this project "
                "at all. Not an undercount: nothing here is being recorded."
                % api_tokens)

    recorded = int(recorded)
    if api_tokens < min_tokens:
        return ("too-little-traffic",
                "%d token(s) in the window, too few for the comparison to mean "
                "anything" % api_tokens)

    gap = api_tokens - recorded
    share = gap / float(api_tokens)
    if share > tolerance:
        return ("undercount",
                "recorded %d token(s) against %d in the org report, short by %d "
                "(%.1f%%). Streamed responses report usage: null unless the "
                "request asked for the totals."
                % (recorded, api_tokens, gap, share * 100))
    if share < -tolerance:
        return ("overcount",
                "recorded %d token(s) against %d in the org report, over by %d "
                "(%.1f%%). Recording more than you were billed for is double "
                "counting, not a streaming gap."
                % (recorded, api_tokens, -gap, -share * 100))
    return ("matched",
            "recorded %d token(s) against %d in the org report (%.1f%% apart)"
            % (recorded, api_tokens, abs(share) * 100))


def untracked_cost(cost_buckets, project_id, api_tokens, gap_tokens):
    """Pro-rata dollars behind an untracked token gap. Pure.

    An estimate and nothing more: input and output are priced differently, so
    scaling a project's spend by its missing token share is only right when the
    missing traffic has the same mix as the rest. It is the right order of
    magnitude and it is read from the cost report rather than a price table,
    which is the most this can honestly claim.
    """
    api_tokens = int(api_tokens or 0)
    gap_tokens = int(gap_tokens or 0)
    if api_tokens <= 0 or gap_tokens <= 0:
        return 0.0
    spend = 0.0
    for bucket in cost_buckets or []:
        for result in bucket.get("results") or []:
            if str(result.get("project_id") or "") != str(project_id):
                continue
            try:
                spend += float((result.get("amount") or {}).get("value") or 0.0)
            except (TypeError, ValueError):
                continue
    return round(spend * min(1.0, gap_tokens / float(api_tokens)), 2)


def get(session, path, params):
    r = session.get(API + path, params=params, timeout=90)
    if r.status_code in (401, 403):
        raise SystemExit("%d from OpenAI: /v1/organization/* needs an "
                         "organization admin key (sk-admin-), not a project key"
                         % r.status_code)
    r.raise_for_status()
    return r.json()


def pages(session, path, params, max_pages=40):
    """Walk a usage or cost report, which paginates on an opaque page cursor."""
    params = dict(params)
    for _ in range(max_pages):
        page = get(session, path, params)
        for bucket in page.get("data") or []:
            yield bucket
        if not page.get("has_more") or not page.get("next_page"):
            return
        params = dict(params)
        params["page"] = page["next_page"]


def main():
    ap = argparse.ArgumentParser(description=__doc__)
    ap.add_argument("--telemetry", required=True,
                    help="JSON file of your own recorded token counts, keyed by "
                         "project id")
    ap.add_argument("--days", type=int, default=7,
                    help="days to reconcile (default 7)")
    ap.add_argument("--tolerance", type=float, default=0.05,
                    help="fractional disagreement to accept as matched "
                         "(default 0.05)")
    ap.add_argument("--min-tokens", type=int, default=100000,
                    help="ignore projects below this many tokens (default 100000)")
    ap.add_argument("--show-all", action="store_true",
                    help="also print projects that reconcile")
    args = ap.parse_args()

    key = os.environ.get("OPENAI_ADMIN_KEY") or os.environ.get("OPENAI_API_KEY")
    if not key:
        log.error("set OPENAI_ADMIN_KEY (an organization admin key, read-only "
                  "scopes are enough)")
        return 2

    try:
        with open(args.telemetry, "r", encoding="utf-8") as fh:
            telemetry = json.load(fh)
    except (OSError, ValueError) as exc:
        log.error("could not read %s: %s", args.telemetry, exc)
        return 2
    if not isinstance(telemetry, dict):
        log.error("%s should be a JSON object keyed by project id", args.telemetry)
        return 2

    session = requests.Session()
    session.headers.update({"Authorization": "Bearer " + key})

    start = int(time.time()) - args.days * 86400
    usage = list(pages(session, "/organization/usage/completions", {
        "start_time": start,
        "bucket_width": "1d",
        "limit": min(31, max(1, args.days)),
        "group_by": ["project_id"],
    }))
    costs = list(pages(session, "/organization/costs", {
        "start_time": start,
        "bucket_width": "1d",
        "limit": min(180, max(1, args.days)),
        "group_by": ["project_id"],
    }))

    rows = api_totals(usage)
    for project in telemetry:
        rows.setdefault(str(project), {"tokens": 0, "requests": 0})
    if not rows:
        log.info("no completions usage in the last %d day(s) and nothing in the "
                 "telemetry file", args.days)
        return 0

    found = 0
    for project in sorted(rows):
        api_tokens = rows[project]["tokens"]
        recorded = recorded_tokens(telemetry.get(project))
        state, detail = compare(api_tokens, recorded, args.tolerance,
                                args.min_tokens)
        line = "%-18s %s  %s" % (state, project, detail)

        if state in FINDINGS:
            found += 1
            log.warning(line)
            if state == "undercount":
                gap = api_tokens - int(recorded or 0)
                money = untracked_cost(costs, project, api_tokens, gap)
                log.warning("  about $%.2f of this project's spend over %d day(s) "
                            "is not in your own numbers", money, args.days)
                log.warning("  repair: set stream_options include_usage on every "
                            "streaming Chat Completions call and read the final "
                            "chunk, or read response.usage from the terminal "
                            "response.completed event on the Responses API. "
                            "Streams the client abandons will still lose theirs.")
            elif state == "overcount":
                log.warning("  repair: this is double counting rather than a "
                            "streaming gap. Look for retries recorded once per "
                            "attempt, or one response written by two consumers.")
            elif state == "untracked":
                log.warning("  repair: this project is absent from your "
                            "telemetry. Map the project id before treating any "
                            "of these numbers as a margin.")
            else:
                log.warning("  repair: your telemetry attributes tokens to a "
                            "project the organization report has no usage for. "
                            "Check the project id, not the streaming client.")
        elif args.show_all:
            log.info(line)

    log.info("%d project(s) reconciled, %d with a gap", len(rows), found)
    return 1 if found else 0


if __name__ == "__main__":
    sys.exit(main())

/**
 * Reconcile OpenAI's token totals against the ones your own telemetry recorded.
 *
 * Read only. Two GET requests against the organization endpoints and a JSON
 * file you supply. Those endpoints reject project keys, so this needs an
 * organization admin key (sk-admin-), provisioned read-only.
 */
import { readFile } from 'node:fs/promises';

const API = 'https://api.openai.com/v1';

const FINDINGS = ['undercount', 'overcount', 'untracked', 'phantom'];

/**
 * Fold usage buckets into one row per project. Pure. Requests ride along
 * because OpenAI reports them and Anthropic does not.
 */
export function apiTotals(buckets) {
  const rows = new Map();
  for (const bucket of buckets ?? []) {
    for (const result of bucket.results ?? []) {
      const project = String(result.project_id ?? 'unknown');
      const row = rows.get(project) ?? { tokens: 0, requests: 0 };
      row.tokens += (Number(result.input_tokens ?? 0) || 0)
                  + (Number(result.output_tokens ?? 0) || 0);
      row.requests += Number(result.num_model_requests ?? 0) || 0;
      rows.set(project, row);
    }
  }
  return rows;
}

/**
 * Read one project's own recorded token count. Pure. Returns a number, or null
 * when nothing was recorded for that project at all. Zero means your pipeline
 * saw the project and recorded nothing; null means it has never heard of it,
 * and those are two different bugs with two different owners.
 */
export function recordedTokens(entry) {
  if (entry === null || entry === undefined) return null;
  if (typeof entry === 'boolean') return null;
  if (typeof entry === 'number') return Number.isFinite(entry) ? Math.trunc(entry) : null;
  if (typeof entry === 'object' && !Array.isArray(entry)) {
    if ('tokens' in entry) {
      const value = Number(entry.tokens ?? 0);
      return Number.isFinite(value) ? Math.trunc(value) : null;
    }
    if ('input_tokens' in entry || 'output_tokens' in entry) {
      const value = (Number(entry.input_tokens ?? 0) || 0)
                  + (Number(entry.output_tokens ?? 0) || 0);
      return Number.isFinite(value) ? Math.trunc(value) : null;
    }
  }
  return null;
}

/**
 * Compare one project's two numbers. Pure. Returns [state, detail]. Three
 * disagreements, not one: short is the streaming gap, over is double counting,
 * and absent from the telemetry is neither.
 */
export function compare(apiTokens, recorded, tolerance = 0.05, minTokens = 100000) {
  const api = Number(apiTokens) || 0;

  if (api <= 0) {
    if (recorded === null || recorded === undefined || Number(recorded) <= 0) {
      return ['idle', 'no usage in the org report and none recorded'];
    }
    return ['phantom',
      `${Math.trunc(Number(recorded))} token(s) recorded against a project the ` +
      'org report shows no usage for. That is a project id mapping, not a ' +
      'streaming problem.'];
  }

  if (recorded === null || recorded === undefined) {
    return ['untracked',
      `${api} token(s) in the org report and no telemetry for this project at ` +
      'all. Not an undercount: nothing here is being recorded.'];
  }

  const seen = Math.trunc(Number(recorded));
  if (api < minTokens) {
    return ['too-little-traffic',
      `${api} token(s) in the window, too few for the comparison to mean anything`];
  }

  const gap = api - seen;
  const share = gap / api;
  if (share > tolerance) {
    return ['undercount',
      `recorded ${seen} token(s) against ${api} in the org report, short by ` +
      `${gap} (${(share * 100).toFixed(1)}%). Streamed responses report usage: ` +
      'null unless the request asked for the totals.'];
  }
  if (share < -tolerance) {
    return ['overcount',
      `recorded ${seen} token(s) against ${api} in the org report, over by ` +
      `${-gap} (${(-share * 100).toFixed(1)}%). Recording more than you were ` +
      'billed for is double counting, not a streaming gap.'];
  }
  return ['matched',
    `recorded ${seen} token(s) against ${api} in the org report ` +
    `(${(Math.abs(share) * 100).toFixed(1)}% apart)`];
}

/**
 * Pro-rata dollars behind an untracked token gap. Pure. An estimate: input and
 * output are priced differently, so this is only exact when the missing traffic
 * has the same mix as the rest. Read from the cost report, not a price table.
 */
export function untrackedCost(costBuckets, projectId, apiTokens, gapTokens) {
  const api = Number(apiTokens) || 0;
  const gap = Number(gapTokens) || 0;
  if (api <= 0 || gap <= 0) return 0;
  let spend = 0;
  for (const bucket of costBuckets ?? []) {
    for (const result of bucket.results ?? []) {
      if (String(result.project_id ?? '') !== String(projectId)) continue;
      spend += Number(result.amount?.value ?? 0) || 0;
    }
  }
  return Math.round(spend * Math.min(1, gap / api) * 100) / 100;
}

async function get(key, path, params) {
  const url = new URL(API + path);
  for (const [k, v] of Object.entries(params)) {
    if (Array.isArray(v)) v.forEach((one) => url.searchParams.append(k, String(one)));
    else if (v !== undefined && v !== null) url.searchParams.set(k, String(v));
  }
  const res = await fetch(url, { headers: { Authorization: `Bearer ${key}` } });
  if (res.status === 401 || res.status === 403) {
    throw new Error(`${res.status} from OpenAI: /v1/organization/* needs an ` +
                    'organization admin key (sk-admin-), not a project key');
  }
  if (!res.ok) throw new Error(`${res.status} from ${path}`);
  return res.json();
}

async function pages(key, path, params, maxPages = 40) {
  const out = [];
  let query = { ...params };
  for (let i = 0; i < maxPages; i += 1) {
    const page = await get(key, path, query);
    out.push(...(page.data ?? []));
    if (!page.has_more || !page.next_page) break;
    query = { ...params, page: page.next_page };
  }
  return out;
}

async function main() {
  const key = process.env.OPENAI_ADMIN_KEY ?? process.env.OPENAI_API_KEY;
  if (!key) {
    console.error('set OPENAI_ADMIN_KEY (an organization admin key, read-only ' +
                  'scopes are enough)');
    process.exitCode = 2;
    return;
  }

  const flag = process.argv.find((a) => a.startsWith('--telemetry='));
  const path = flag ? flag.slice('--telemetry='.length) : process.env.TELEMETRY;
  if (!path) {
    console.error('pass --telemetry=week.json (your own recorded token counts, ' +
                  'keyed by project id)');
    process.exitCode = 2;
    return;
  }

  let telemetry;
  try {
    telemetry = JSON.parse(await readFile(path, 'utf8'));
  } catch (err) {
    console.error(`could not read ${path}: ${err.message}`);
    process.exitCode = 2;
    return;
  }
  if (telemetry === null || typeof telemetry !== 'object' || Array.isArray(telemetry)) {
    console.error(`${path} should be a JSON object keyed by project id`);
    process.exitCode = 2;
    return;
  }

  const days = Number(process.env.DAYS ?? 7);
  const tolerance = Number(process.env.TOLERANCE ?? 0.05);
  const minTokens = Number(process.env.MIN_TOKENS ?? 100000);
  const showAll = process.argv.includes('--show-all');

  const start = Math.floor(Date.now() / 1000) - days * 86400;
  const usage = await pages(key, '/organization/usage/completions', {
    start_time: start,
    bucket_width: '1d',
    limit: Math.min(31, Math.max(1, days)),
    group_by: ['project_id'],
  });
  const costs = await pages(key, '/organization/costs', {
    start_time: start,
    bucket_width: '1d',
    limit: Math.min(180, Math.max(1, days)),
    group_by: ['project_id'],
  });

  const rows = apiTotals(usage);
  for (const project of Object.keys(telemetry)) {
    if (!rows.has(project)) rows.set(project, { tokens: 0, requests: 0 });
  }
  if (rows.size === 0) {
    console.log(`no completions usage in the last ${days} day(s) and nothing in ` +
                'the telemetry file');
    return;
  }

  let found = 0;
  for (const project of [...rows.keys()].sort()) {
    const apiTokens = rows.get(project).tokens;
    const recorded = recordedTokens(telemetry[project]);
    const [state, detail] = compare(apiTokens, recorded, tolerance, minTokens);
    const line = `${state.padEnd(18)} ${project}  ${detail}`;

    if (FINDINGS.includes(state)) {
      found += 1;
      console.warn(line);
      if (state === 'undercount') {
        const gap = apiTokens - (recorded ?? 0);
        const money = untrackedCost(costs, project, apiTokens, gap);
        console.warn(`  about $${money.toFixed(2)} of this project's spend over ` +
          `${days} day(s) is not in your own numbers`);
        console.warn('  repair: set stream_options include_usage on every ' +
          'streaming Chat Completions call and read the final chunk, or read ' +
          'response.usage from the terminal response.completed event on the ' +
          'Responses API. Streams the client abandons will still lose theirs.');
      } else if (state === 'overcount') {
        console.warn('  repair: this is double counting rather than a streaming ' +
          'gap. Look for retries recorded once per attempt, or one response ' +
          'written by two consumers.');
      } else if (state === 'untracked') {
        console.warn('  repair: this project is absent from your telemetry. Map ' +
          'the project id before treating any of these numbers as a margin.');
      } else {
        console.warn('  repair: your telemetry attributes tokens to a project ' +
          'the organization report has no usage for. Check the project id, not ' +
          'the streaming client.');
      }
    } else if (showAll) {
      console.log(line);
    }
  }

  console.log(`${rows.size} project(s) reconciled, ${found} with a gap`);
  process.exitCode = found ? 1 : 0;
}

// Only run when invoked directly, so importing this module from the test file
// does not fire main() and fail on the missing key.
if (import.meta.url === `file://${process.argv[1]}`) {
  main().catch((err) => { console.error(err.message); process.exitCode = 2; });
}

from openai_streaming_usage_gap import (api_totals, compare, recorded_tokens,
                                          untracked_cost)


def bucket(*results):
    return {"start_time": 0, "end_time": 86400, "results": list(results)}


def usage(project="proj_chat", input_tokens=0, output_tokens=0, requests=0):
    return {"project_id": project, "input_tokens": input_tokens,
            "output_tokens": output_tokens, "num_model_requests": requests}


def test_a_dashboard_short_of_the_org_report_is_the_finding():
    state, detail = compare(api_tokens=42_000_000, recorded=28_000_000)
    assert state == "undercount"
    assert "short by 14000000" in detail
    assert "33.3%" in detail
    assert "usage: null" in detail


def test_recording_more_than_you_were_billed_for_is_a_different_bug():
    # An absolute-value comparison would call this an undercount and send
    # somebody to add a streaming parameter that changes nothing.
    state, detail = compare(api_tokens=10_000_000, recorded=13_000_000)
    assert state == "overcount"
    assert "double counting" in detail


def test_a_project_missing_from_telemetry_is_not_an_undercount():
    state, detail = compare(api_tokens=9_000_000, recorded=None)
    assert state == "untracked"
    assert "nothing here is being recorded" in detail
    # Recorded as zero is a different sentence: the pipeline saw it.
    assert compare(api_tokens=9_000_000, recorded=0)[0] == "undercount"


def test_tokens_recorded_against_a_project_with_no_usage_are_a_mapping_bug():
    state, detail = compare(api_tokens=0, recorded=5_000_000)
    assert state == "phantom"
    assert "project id mapping" in detail
    assert compare(api_tokens=0, recorded=None)[0] == "idle"
    assert compare(api_tokens=0, recorded=0)[0] == "idle"


def test_small_projects_and_close_numbers_are_not_findings():
    assert compare(api_tokens=5_000, recorded=1)[0] == "too-little-traffic"
    state, detail = compare(api_tokens=1_000_000, recorded=980_000)
    assert state == "matched"
    assert "2.0% apart" in detail


def test_usage_buckets_fold_into_one_row_per_project():
    rows = api_totals([
        bucket(usage(input_tokens=100, output_tokens=20, requests=3),
               usage(project="proj_batch", input_tokens=7, output_tokens=1)),
        bucket(usage(input_tokens=50, output_tokens=5, requests=2)),
    ])
    assert rows["proj_chat"] == {"tokens": 175, "requests": 5}
    assert rows["proj_batch"] == {"tokens": 8, "requests": 0}


def test_telemetry_is_read_leniently_but_absence_is_preserved():
    assert recorded_tokens(1200) == 1200
    assert recorded_tokens({"tokens": 1200}) == 1200
    assert recorded_tokens({"input_tokens": 900, "output_tokens": 300}) == 1200
    assert recorded_tokens(0) == 0
    assert recorded_tokens(None) is None
    assert recorded_tokens({}) is None
    assert recorded_tokens("lots") is None
    assert recorded_tokens(True) is None


def test_the_money_is_a_pro_rata_share_of_reported_spend():
    costs = [bucket({"project_id": "proj_chat",
                     "amount": {"value": 300.0, "currency": "usd"}},
                    {"project_id": "proj_other",
                     "amount": {"value": 900.0, "currency": "usd"}})]
    assert untracked_cost(costs, "proj_chat", 1_000_000, 250_000) == 75.0
    assert untracked_cost(costs, "proj_chat", 1_000_000, 0) == 0.0
    assert untracked_cost(costs, "proj_chat", 0, 100) == 0.0
    assert untracked_cost(costs, "proj_missing", 1_000_000, 500_000) == 0.0

import { test } from 'node:test';
import assert from 'node:assert/strict';
import { apiTotals, compare, recordedTokens, untrackedCost }
  from './openai-streaming-usage-gap.mjs';

function bucket(...results) {
  return { start_time: 0, end_time: 86400, results };
}

function usage({ project = 'proj_chat', inputTokens = 0, outputTokens = 0,
                 requests = 0 } = {}) {
  return { project_id: project, input_tokens: inputTokens,
           output_tokens: outputTokens, num_model_requests: requests };
}

test('a dashboard short of the org report is the finding', () => {
  const [state, detail] = compare(42000000, 28000000);
  assert.equal(state, 'undercount');
  assert.match(detail, /short by 14000000/);
  assert.match(detail, /33\.3%/);
  assert.match(detail, /usage: null/);
});

test('recording more than you were billed for is a different bug', () => {
  const [state, detail] = compare(10000000, 13000000);
  assert.equal(state, 'overcount');
  assert.match(detail, /double counting/);
});

test('a project missing from telemetry is not an undercount', () => {
  const [state, detail] = compare(9000000, null);
  assert.equal(state, 'untracked');
  assert.match(detail, /nothing here is being recorded/);
  assert.equal(compare(9000000, 0)[0], 'undercount');
});

test('tokens recorded against a project with no usage are a mapping bug', () => {
  const [state, detail] = compare(0, 5000000);
  assert.equal(state, 'phantom');
  assert.match(detail, /project id mapping/);
  assert.equal(compare(0, null)[0], 'idle');
  assert.equal(compare(0, 0)[0], 'idle');
});

test('small projects and close numbers are not findings', () => {
  assert.equal(compare(5000, 1)[0], 'too-little-traffic');
  const [state, detail] = compare(1000000, 980000);
  assert.equal(state, 'matched');
  assert.match(detail, /2\.0% apart/);
});

test('usage buckets fold into one row per project', () => {
  const rows = apiTotals([
    bucket(usage({ inputTokens: 100, outputTokens: 20, requests: 3 }),
           usage({ project: 'proj_batch', inputTokens: 7, outputTokens: 1 })),
    bucket(usage({ inputTokens: 50, outputTokens: 5, requests: 2 })),
  ]);
  assert.deepEqual(rows.get('proj_chat'), { tokens: 175, requests: 5 });
  assert.deepEqual(rows.get('proj_batch'), { tokens: 8, requests: 0 });
});

test('telemetry is read leniently but absence is preserved', () => {
  assert.equal(recordedTokens(1200), 1200);
  assert.equal(recordedTokens({ tokens: 1200 }), 1200);
  assert.equal(recordedTokens({ input_tokens: 900, output_tokens: 300 }), 1200);
  assert.equal(recordedTokens(0), 0);
  assert.equal(recordedTokens(null), null);
  assert.equal(recordedTokens({}), null);
  assert.equal(recordedTokens('lots'), null);
  assert.equal(recordedTokens(true), null);
});

test('the money is a pro rata share of reported spend', () => {
  const costs = [bucket(
    { project_id: 'proj_chat', amount: { value: 300.0, currency: 'usd' } },
    { project_id: 'proj_other', amount: { value: 900.0, currency: 'usd' } },
  )];
  assert.equal(untrackedCost(costs, 'proj_chat', 1000000, 250000), 75);
  assert.equal(untrackedCost(costs, 'proj_chat', 1000000, 0), 0);
  assert.equal(untrackedCost(costs, 'proj_chat', 0, 100), 0);
  assert.equal(untrackedCost(costs, 'proj_missing', 1000000, 500000), 0);
});