a fine-tuned model was trained, billed, and never called once

"""Report OpenAI fine-tuned models that were trained, billed, and never called.

Read only. GET requests and nothing else, and it needs two credentials because
no single key can answer the question:

  OPENAI_API_KEY    a project key set to Read Only, for /v1/fine_tuning/jobs,
                    /v1/models and /v1/files
  OPENAI_ADMIN_KEY  an organization admin key with read scopes, for
                    /v1/organization/usage/completions

The repair is printed, never performed. Deleting a custom model somebody spent
a quarter producing is a decision with an owner, and that owner is not a cron.
"""
import argparse
import datetime as dt
import logging
import os
import sys

import requests

logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s")
log = logging.getLogger("openai_fine_tune_usage_audit")

API = "https://api.openai.com/v1"

# Published platform dates. Fine-tuned snapshots built on a retired base model
# stop answering on the first; new fine-tuning jobs cannot be created after the
# second. Both are printed rather than acted on.
BASE_RETIREMENT = "2026-10-23"
NEW_JOBS_BLOCKED = "2027-01-06"

FINDINGS = ("never-called", "never-called-base-gone", "in-service-base-gone")


def base_model(fine_tuned_model):
    """The base model a fine-tune id was built on, or None. Pure.

    "ft:gpt-4o-mini-2024-07-18:acme::AbC123" -> "gpt-4o-mini-2024-07-18". The
    optional suffix segment moves the trailing id along, so this reads the
    second field rather than counting from the end.
    """
    name = str(fine_tuned_model or "").strip()
    if not name.lower().startswith("ft:"):
        return None
    parts = name.split(":")
    if len(parts) < 3 or not parts[1]:
        return None
    return parts[1]


def days_until(date_str, now):
    """Whole days from now until an ISO date, or None if unreadable. Pure.

    Negative once the date has passed. Floored toward the past, so a deadline
    fourteen hours away reads as 0 days rather than 1: this number is printed to
    somebody who will act on it tomorrow.
    """
    try:
        year, month, day = (int(p) for p in str(date_str).split("-"))
        target = dt.datetime(year, month, day, tzinfo=dt.timezone.utc)
    except (TypeError, ValueError):
        return None
    return int((target - now).total_seconds() // 86400)


def verdict(job, requests_made, available_models, now, window_days=30):
    """Classify one fine-tuning job against its usage. Pure. Returns (state, detail).

    available_models is the set of ids GET /v1/models returned, which is what the
    key can actually call. A base missing from it puts a deadline on the custom
    model whether or not anyone is using it, so that case is split out rather
    than folded into the idle one.
    """
    status = str(job.get("status") or "").strip().lower()
    if status != "succeeded":
        return ("not-succeeded",
                "status is %s, so there is no model id to look for usage against"
                % (status or "missing"))

    model_id = str(job.get("fine_tuned_model") or "").strip()
    if not model_id:
        return ("unnamed",
                "the job succeeded and carries no fine_tuned_model. Read the "
                "object by hand rather than assuming nothing was produced.")

    try:
        trained = int(job.get("trained_tokens") or 0)
    except (TypeError, ValueError):
        trained = 0
    try:
        calls = int(requests_made or 0)
    except (TypeError, ValueError):
        calls = 0

    base = job.get("model") or base_model(model_id)
    base_gone = bool(base) and base not in set(available_models or ())
    deadline = days_until(BASE_RETIREMENT, now)
    clock = ("" if deadline is None else
             " Fine-tunes on retired base models stop answering in %d day(s)."
             % deadline)

    if calls > 0:
        if base_gone:
            return ("in-service-base-gone",
                    "%d request(s) in %d days, but the base model %s is no "
                    "longer listed by GET /v1/models. This fine-tune is serving "
                    "traffic and is going to stop.%s"
                    % (calls, window_days, base, clock))
        return ("in-service",
                "%d request(s) in %d days" % (calls, window_days))

    if base_gone:
        return ("never-called-base-gone",
                "0 request(s) in %d days, %d trained token(s), and the base "
                "model %s is no longer listed. Nothing to migrate and nothing "
                "to lose.%s" % (window_days, trained, base, clock))

    return ("never-called",
            "0 request(s) in %d days, %d trained token(s). Training was billed "
            "and inference never happened." % (window_days, trained))


def get(session, path, params=None):
    r = session.get(API + path, params=params or {}, timeout=60)
    if r.status_code == 401:
        raise SystemExit("401 from OpenAI on %s: wrong key for this endpoint. "
                         "Jobs, models and files want the project key; usage "
                         "wants the admin key." % path)
    r.raise_for_status()
    return r.json()


def jobs(session, max_pages=20):
    """Walk GET /v1/fine_tuning/jobs, which paginates on the last job's id."""
    params = {"limit": 100}
    for _ in range(max_pages):
        page = get(session, "/fine_tuning/jobs", params)
        data = page.get("data") or []
        for job in data:
            yield job
        if not page.get("has_more") or not data:
            return
        params = {"limit": 100, "after": data[-1].get("id")}


def requests_by_model(session, start_time, days, max_pages=20):
    """Summed num_model_requests per model id. Needs the admin key."""
    out = {}
    params = {"start_time": start_time, "bucket_width": "1d", "limit": days,
              "group_by": "model"}
    for _ in range(max_pages):
        page = get(session, "/organization/usage/completions", params)
        for bucket in page.get("data") or []:
            for result in bucket.get("results") or []:
                model = str(result.get("model") or "")
                if not model:
                    continue
                try:
                    out[model] = out.get(model, 0) + int(
                        result.get("num_model_requests") or 0)
                except (TypeError, ValueError):
                    pass
        cursor = page.get("next_page")
        if not cursor:
            return out
        params = dict(params, page=cursor)
    return out


def available_model_ids(session):
    page = get(session, "/models")
    return {str(m.get("id")) for m in page.get("data") or [] if m.get("id")}


def result_file_bytes(session):
    """Total bytes still held by fine-tune result files, and how many there are."""
    page = get(session, "/files", {"purpose": "fine-tune-results", "limit": 100})
    files = page.get("data") or []
    total = 0
    for f in files:
        try:
            total += int(f.get("bytes") or 0)
        except (TypeError, ValueError):
            pass
    return len(files), total


def main():
    ap = argparse.ArgumentParser(description=__doc__)
    ap.add_argument("--days", type=int, default=30,
                    help="usage window in days (default 30)")
    ap.add_argument("--show-all", action="store_true",
                    help="also print jobs that are in service or not succeeded")
    args = ap.parse_args()

    project_key = os.environ.get("OPENAI_API_KEY")
    admin_key = os.environ.get("OPENAI_ADMIN_KEY")
    if not project_key or not admin_key:
        log.error("set OPENAI_API_KEY (a project key set to Read Only) and "
                  "OPENAI_ADMIN_KEY (an organization admin key with read scopes)")
        return 2

    project = requests.Session()
    project.headers.update({"Authorization": "Bearer " + project_key})
    admin = requests.Session()
    admin.headers.update({"Authorization": "Bearer " + admin_key})

    now = dt.datetime.now(dt.timezone.utc)
    start = int((now - dt.timedelta(days=args.days)).timestamp())

    usage = requests_by_model(admin, start, args.days)
    available = available_model_ids(project)

    checked = 0
    bad = 0
    for job in jobs(project):
        model_id = str(job.get("fine_tuned_model") or "")
        state, detail = verdict(job, usage.get(model_id, 0), available, now,
                                args.days)
        if state != "not-succeeded":
            checked += 1
        line = "%-22s %-42s %s" % (state, model_id or job.get("id"), detail)

        if state in FINDINGS:
            bad += 1
            log.warning(line)
            checkpoints = get(project, "/fine_tuning/jobs/%s/checkpoints"
                              % job.get("id")).get("data") or []
            for cp in checkpoints:
                cp_id = cp.get("fine_tuned_model_checkpoint")
                if cp_id:
                    log.warning("  checkpoint %s: %d request(s) in the window",
                                cp_id, usage.get(str(cp_id), 0))
            log.warning("  repair: route traffic to it or retire it. Deleting "
                        "the custom model and its result_files stops the "
                        "storage charge; GET /v1/files?purpose=fine-tune-results "
                        "lists them.")
            left = days_until(NEW_JOBS_BLOCKED, now)
            if left is not None:
                log.warning("  repair: decide before the platform decides. New "
                            "fine-tuning jobs cannot be created after %s, %d "
                            "day(s) away.", NEW_JOBS_BLOCKED, left)
        elif args.show_all:
            log.info(line)

    count, total_bytes = result_file_bytes(project)
    if count:
        log.info("%d fine-tune result file(s) still stored, %.1f MB",
                 count, total_bytes / 1048576.0)

    log.info("%d succeeded job(s) checked, %d finding(s)", checked, bad)
    return 1 if bad else 0


if __name__ == "__main__":
    sys.exit(main())

/**
 * Report OpenAI fine-tuned models that were trained, billed, and never called.
 *
 * Read only. GET requests and nothing else, and it needs two credentials
 * because no single key can answer the question:
 *
 *   OPENAI_API_KEY    a project key set to Read Only, for /v1/fine_tuning/jobs,
 *                     /v1/models and /v1/files
 *   OPENAI_ADMIN_KEY  an organization admin key with read scopes, for
 *                     /v1/organization/usage/completions
 *
 * The repair is printed, never performed.
 */
const API = 'https://api.openai.com/v1';

// Published platform dates. Fine-tuned snapshots on a retired base model stop
// answering on the first; new fine-tuning jobs cannot be created after the second.
const BASE_RETIREMENT = '2026-10-23';
const NEW_JOBS_BLOCKED = '2027-01-06';

const FINDINGS = ['never-called', 'never-called-base-gone', 'in-service-base-gone'];

/**
 * The base model a fine-tune id was built on, or null. Pure.
 * "ft:gpt-4o-mini-2024-07-18:acme::AbC123" -> "gpt-4o-mini-2024-07-18".
 */
export function baseModel(fineTunedModel) {
  const name = String(fineTunedModel ?? '').trim();
  if (!name.toLowerCase().startsWith('ft:')) return null;
  const parts = name.split(':');
  if (parts.length < 3 || !parts[1]) return null;
  return parts[1];
}

/**
 * Whole days from now until an ISO date, or null if unreadable. Pure.
 * Negative once the date has passed, and floored toward the past so a deadline
 * fourteen hours away reads as 0 days rather than 1.
 */
export function daysUntil(dateStr, now) {
  const parts = String(dateStr ?? '').split('-');
  if (parts.length !== 3) return null;
  const [year, month, day] = parts.map((p) => Number(p));
  if (![year, month, day].every(Number.isFinite)) return null;
  const target = Date.UTC(year, month - 1, day);
  const from = now instanceof Date ? now.getTime() : Number(now);
  if (!Number.isFinite(from)) return null;
  return Math.floor((target - from) / 86400000);
}

/**
 * Classify one fine-tuning job against its usage. Pure. Returns [state, detail].
 * A base model missing from GET /v1/models puts a deadline on the custom model
 * whether or not anyone is using it, so that case is split out.
 */
export function verdict(job, requestsMade, availableModels, now, windowDays = 30) {
  const status = String(job.status ?? '').trim().toLowerCase();
  if (status !== 'succeeded') {
    return ['not-succeeded',
      `status is ${status || 'missing'}, so there is no model id to look for ` +
      'usage against'];
  }

  const modelId = String(job.fine_tuned_model ?? '').trim();
  if (!modelId) {
    return ['unnamed',
      'the job succeeded and carries no fine_tuned_model. Read the object by ' +
      'hand rather than assuming nothing was produced.'];
  }

  const trainedRaw = Number(job.trained_tokens ?? 0);
  const trained = Number.isFinite(trainedRaw) ? Math.trunc(trainedRaw) : 0;
  const callsRaw = Number(requestsMade ?? 0);
  const calls = Number.isFinite(callsRaw) ? Math.trunc(callsRaw) : 0;

  const base = job.model ?? baseModel(modelId);
  const available = new Set(availableModels ?? []);
  const baseGone = Boolean(base) && !available.has(base);
  const deadline = daysUntil(BASE_RETIREMENT, now);
  const clock = deadline === null ? ''
    : ` Fine-tunes on retired base models stop answering in ${deadline} day(s).`;

  if (calls > 0) {
    if (baseGone) {
      return ['in-service-base-gone',
        `${calls} request(s) in ${windowDays} days, but the base model ${base} ` +
        'is no longer listed by GET /v1/models. This fine-tune is serving ' +
        `traffic and is going to stop.${clock}`];
    }
    return ['in-service', `${calls} request(s) in ${windowDays} days`];
  }

  if (baseGone) {
    return ['never-called-base-gone',
      `0 request(s) in ${windowDays} days, ${trained} trained token(s), and ` +
      `the base model ${base} is no longer listed. Nothing to migrate and ` +
      `nothing to lose.${clock}`];
  }

  return ['never-called',
    `0 request(s) in ${windowDays} days, ${trained} trained token(s). Training ` +
    'was billed and inference never happened.'];
}

async function get(key, path, params = {}) {
  const url = new URL(API + path);
  for (const [k, v] of Object.entries(params)) {
    if (v !== undefined && v !== null) url.searchParams.set(k, String(v));
  }
  const res = await fetch(url, { headers: { Authorization: `Bearer ${key}` } });
  if (res.status === 401) {
    throw new Error(`401 from OpenAI on ${path}: wrong key for this endpoint. ` +
      'Jobs, models and files want the project key; usage wants the admin key.');
  }
  if (!res.ok) throw new Error(`${res.status} from ${path}`);
  return res.json();
}

async function* walkJobs(key, maxPages = 20) {
  let params = { limit: 100 };
  for (let i = 0; i < maxPages; i += 1) {
    const page = await get(key, '/fine_tuning/jobs', params);
    const data = page.data ?? [];
    for (const job of data) yield job;
    if (!page.has_more || data.length === 0) return;
    params = { limit: 100, after: data[data.length - 1].id };
  }
}

async function requestsByModel(key, startTime, days, maxPages = 20) {
  const out = {};
  let params = {
    start_time: startTime, bucket_width: '1d', limit: days, group_by: 'model',
  };
  for (let i = 0; i < maxPages; i += 1) {
    const page = await get(key, '/organization/usage/completions', params);
    for (const bucket of page.data ?? []) {
      for (const result of bucket.results ?? []) {
        const model = String(result.model ?? '');
        if (!model) continue;
        const n = Number(result.num_model_requests ?? 0);
        if (Number.isFinite(n)) out[model] = (out[model] ?? 0) + Math.trunc(n);
      }
    }
    if (!page.next_page) return out;
    params = { ...params, page: page.next_page };
  }
  return out;
}

async function availableModelIds(key) {
  const page = await get(key, '/models');
  return new Set((page.data ?? []).filter((m) => m.id).map((m) => String(m.id)));
}

async function resultFileBytes(key) {
  const page = await get(key, '/files', { purpose: 'fine-tune-results', limit: 100 });
  const files = page.data ?? [];
  let total = 0;
  for (const f of files) {
    const n = Number(f.bytes ?? 0);
    if (Number.isFinite(n)) total += Math.trunc(n);
  }
  return [files.length, total];
}

async function main() {
  const projectKey = process.env.OPENAI_API_KEY;
  const adminKey = process.env.OPENAI_ADMIN_KEY;
  if (!projectKey || !adminKey) {
    console.error('set OPENAI_API_KEY (a project key set to Read Only) and ' +
                  'OPENAI_ADMIN_KEY (an organization admin key with read scopes)');
    process.exitCode = 2;
    return;
  }

  const days = Number(process.env.DAYS ?? 30);
  const showAll = process.argv.includes('--show-all');

  const nowMs = Date.now();
  const now = new Date(nowMs);
  const start = Math.floor(nowMs / 1000) - days * 86400;

  const usage = await requestsByModel(adminKey, start, days);
  const available = await availableModelIds(projectKey);

  let checked = 0;
  let bad = 0;
  for await (const job of walkJobs(projectKey)) {
    const modelId = String(job.fine_tuned_model ?? '');
    const [state, detail] = verdict(job, usage[modelId] ?? 0, available, now, days);
    if (state !== 'not-succeeded') checked += 1;
    const line = `${state.padEnd(22)} ${(modelId || job.id).padEnd(42)} ${detail}`;

    if (FINDINGS.includes(state)) {
      bad += 1;
      console.warn(line);
      const page = await get(projectKey, `/fine_tuning/jobs/${job.id}/checkpoints`);
      for (const cp of page.data ?? []) {
        const cpId = cp.fine_tuned_model_checkpoint;
        if (cpId) {
          console.warn(`  checkpoint ${cpId}: ${usage[String(cpId)] ?? 0} ` +
                       'request(s) in the window');
        }
      }
      console.warn('  repair: route traffic to it or retire it. Deleting the ' +
        'custom model and its result_files stops the storage charge; ' +
        'GET /v1/files?purpose=fine-tune-results lists them.');
      const left = daysUntil(NEW_JOBS_BLOCKED, now);
      if (left !== null) {
        console.warn('  repair: decide before the platform decides. New ' +
          `fine-tuning jobs cannot be created after ${NEW_JOBS_BLOCKED}, ` +
          `${left} day(s) away.`);
      }
    } else if (showAll) {
      console.log(line);
    }
  }

  const [count, totalBytes] = await resultFileBytes(projectKey);
  if (count) {
    console.log(`${count} fine-tune result file(s) still stored, ` +
                `${(totalBytes / 1048576).toFixed(1)} MB`);
  }

  console.log(`${checked} succeeded job(s) checked, ${bad} finding(s)`);
  process.exitCode = bad ? 1 : 0;
}

// Only run when invoked directly, so importing this from the test file does not
// run main(), fail on the missing keys, and set an exit code that fails the suite.
if (import.meta.url === `file://${process.argv[1]}`) {
  main().catch((err) => { console.error(err.message); process.exitCode = 2; });
}

import datetime as dt

from openai_fine_tune_usage_audit import base_model, days_until, verdict

NOW = dt.datetime(2026, 8, 30, 12, 0, tzinfo=dt.timezone.utc)
LIVE = {"gpt-4o-mini-2024-07-18", "gpt-5", "gpt-5-mini"}


def job(status="succeeded", model_id="ft:gpt-4o-mini-2024-07-18:acme::AbC123",
        base="gpt-4o-mini-2024-07-18", trained=4182900, **extra):
    body = {"id": "ftjob-test", "status": status, "fine_tuned_model": model_id,
            "model": base, "trained_tokens": trained}
    body.update(extra)
    return body


def test_trained_billed_and_never_called():
    state, detail = verdict(job(), 0, LIVE, NOW)
    assert state == "never-called"
    assert "0 request(s) in 30 days" in detail
    assert "4182900 trained token(s)" in detail


def test_a_model_serving_traffic_is_not_a_finding():
    assert verdict(job(), 91204, LIVE, NOW)[0] == "in-service"


def test_a_vanished_base_model_changes_both_answers():
    # Idle on a base that is going away: nothing to migrate, delete it.
    state, detail = verdict(job(base="gpt-4-0613",
                                model_id="ft:gpt-4-0613:acme::Old1"),
                            0, LIVE, NOW)
    assert state == "never-called-base-gone"
    assert "no longer listed" in detail
    assert "stop answering in 53 day(s)" in detail

    # In service on a base that is going away: this one is urgent.
    state, detail = verdict(job(base="gpt-4-0613",
                                model_id="ft:gpt-4-0613:acme::Old1"),
                            50000, LIVE, NOW)
    assert state == "in-service-base-gone"
    assert "going to stop" in detail


def test_jobs_that_produced_nothing_are_not_this_note():
    assert verdict(job(status="failed"), 0, LIVE, NOW)[0] == "not-succeeded"
    assert verdict(job(status="running"), 0, LIVE, NOW)[0] == "not-succeeded"
    assert verdict(job(status="cancelled"), 0, LIVE, NOW)[0] == "not-succeeded"
    state, detail = verdict(job(model_id=None), 0, LIVE, NOW)
    assert state == "unnamed"
    assert "by hand" in detail


def test_the_base_is_the_second_field_not_the_last_one():
    assert base_model("ft:gpt-4o-mini-2024-07-18:acme::AbC123") == "gpt-4o-mini-2024-07-18"
    # An optional suffix moves the trailing id along; the base does not move.
    assert base_model("ft:gpt-4o-2024-08-06:acme:nightly:AbC123") == "gpt-4o-2024-08-06"
    assert base_model("gpt-5") is None
    assert base_model("") is None
    assert base_model(None) is None


def test_the_deadline_is_floored_toward_the_past():
    assert days_until("2026-10-23", NOW) == 53
    # 12 hours short of the date is 0 days left, not 1.
    assert days_until("2026-08-31", NOW) == 0
    assert days_until("2026-08-30", NOW) == -1
    assert days_until("not-a-date", NOW) is None


def test_a_job_with_no_base_field_falls_back_to_the_model_id():
    # Some job objects carry the base only inside fine_tuned_model.
    state, _ = verdict({"id": "ftjob-x", "status": "succeeded",
                        "fine_tuned_model": "ft:gpt-4-0613:acme::Old1",
                        "trained_tokens": 100}, 0, LIVE, NOW)
    assert state == "never-called-base-gone"

import { test } from 'node:test';
import assert from 'node:assert/strict';
import { baseModel, daysUntil, verdict }
  from './openai-fine-tune-usage-audit.mjs';

const NOW = new Date(Date.UTC(2026, 7, 30, 12, 0, 0));
const LIVE = ['gpt-4o-mini-2024-07-18', 'gpt-5', 'gpt-5-mini'];

function job({ status = 'succeeded',
               modelId = 'ft:gpt-4o-mini-2024-07-18:acme::AbC123',
               base = 'gpt-4o-mini-2024-07-18', trained = 4182900,
               ...extra } = {}) {
  return {
    id: 'ftjob-test', status, fine_tuned_model: modelId, model: base,
    trained_tokens: trained, ...extra,
  };
}

test('trained, billed and never called', () => {
  const [state, detail] = verdict(job(), 0, LIVE, NOW);
  assert.equal(state, 'never-called');
  assert.match(detail, /0 request.s. in 30 days/);
  assert.match(detail, /4182900 trained token/);
});

test('a model serving traffic is not a finding', () => {
  assert.equal(verdict(job(), 91204, LIVE, NOW)[0], 'in-service');
});

test('a vanished base model changes both answers', () => {
  const idle = verdict(job({ base: 'gpt-4-0613', modelId: 'ft:gpt-4-0613:acme::Old1' }),
                       0, LIVE, NOW);
  assert.equal(idle[0], 'never-called-base-gone');
  assert.match(idle[1], /no longer listed/);
  assert.match(idle[1], /stop answering in 53 day/);

  const live = verdict(job({ base: 'gpt-4-0613', modelId: 'ft:gpt-4-0613:acme::Old1' }),
                       50000, LIVE, NOW);
  assert.equal(live[0], 'in-service-base-gone');
  assert.match(live[1], /going to stop/);
});

test('jobs that produced nothing are not this note', () => {
  for (const status of ['failed', 'running', 'cancelled']) {
    assert.equal(verdict(job({ status }), 0, LIVE, NOW)[0], 'not-succeeded');
  }
  const [state, detail] = verdict(job({ modelId: null }), 0, LIVE, NOW);
  assert.equal(state, 'unnamed');
  assert.match(detail, /by hand/);
});

test('the base is the second field not the last one', () => {
  assert.equal(baseModel('ft:gpt-4o-mini-2024-07-18:acme::AbC123'),
               'gpt-4o-mini-2024-07-18');
  assert.equal(baseModel('ft:gpt-4o-2024-08-06:acme:nightly:AbC123'),
               'gpt-4o-2024-08-06');
  assert.equal(baseModel('gpt-5'), null);
  assert.equal(baseModel(''), null);
  assert.equal(baseModel(null), null);
});

test('the deadline is floored toward the past', () => {
  assert.equal(daysUntil('2026-10-23', NOW), 53);
  assert.equal(daysUntil('2026-08-31', NOW), 0);
  assert.equal(daysUntil('2026-08-30', NOW), -1);
  assert.equal(daysUntil('not-a-date', NOW), null);
});

test('a job with no base field falls back to the model id', () => {
  const [state] = verdict({
    id: 'ftjob-x', status: 'succeeded',
    fine_tuned_model: 'ft:gpt-4-0613:acme::Old1', trained_tokens: 100,
  }, 0, LIVE, NOW);
  assert.equal(state, 'never-called-base-gone');
});