The fine-tuning job failed and error.code was never read

"""Find fine-tuning jobs that were accepted, then failed, and never read.

Read only. GET /v1/fine_tuning/jobs, paginated, plus GET on the events feed for
jobs that failed. Nothing is created, cancelled, deleted or re-uploaded.

Job creation is asynchronous. The create call returns 200 as soon as the job is
accepted, and validation and training failures surface only on the job object:
status becomes failed, fine_tuned_model and trained_tokens stay null, and error
carries code, message and param. None of that is pushed anywhere.

The error codes are an open set. The documented ones are translated into an
action; everything else is printed exactly as returned, because inventing an
interpretation sends somebody confidently in a direction the API never suggested.

Scope: this note owns a job that failed. Whether a job that succeeded is ever
called is fine-tuned-model-never-used, and whether new jobs can be created at
all is a platform question about dates. Neither is read here.
"""
import argparse
import logging
import os
import sys
import time

import requests

logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s")
log = logging.getLogger("openai_fine_tune_failures")

JOBS_URL = "https://api.openai.com/v1/fine_tuning/jobs"

TERMINAL = ("succeeded", "failed", "cancelled")

# Only the codes with a documented meaning. Anything else is printed verbatim.
ADVICE = {
    "invalid_training_file":
        "the JSONL is malformed. One JSON object per line, no trailing blank "
        "line, no BOM, each row a messages array with at least one assistant "
        "turn, and one schema across every row.",
    "invalid_validation_file":
        "the validation file has the same problem as a malformed training "
        "file, and error.param says which of the two was rejected.",
    "invalid_n_examples":
        "the example count is out of range: too few rows to train on, or more "
        "than the method accepts. Count the lines before uploading.",
    "exceeded_quota":
        "this is a billing problem rather than a data one. Editing the file "
        "will not help; check the account's quota and spend limits.",
}

FINDINGS = ("job-failed", "failed-without-error", "stalled-in-validation")


def job_row(body):
    """One job, reduced. Pure. The error object is flattened.

    Flattened deliberately: a job with no error key and a job with an empty
    error object mean the same thing to a reader and should not need two code
    paths to say so.
    """
    body = body if isinstance(body, dict) else {}
    error = body.get("error") if isinstance(body.get("error"), dict) else {}
    try:
        created = int(body.get("created_at") or 0)
    except (TypeError, ValueError):
        created = 0
    return {"id": str(body.get("id") or ""),
            "status": str(body.get("status") or ""),
            "model": str(body.get("model") or ""),
            "fine_tuned_model": str(body.get("fine_tuned_model") or ""),
            "created_at": created,
            "code": str((error or {}).get("code") or ""),
            "param": str((error or {}).get("param") or ""),
            "message": str((error or {}).get("message") or "")}


def hours_since(created_at, now):
    """Age in hours. Pure. The clock is an argument."""
    try:
        created = int(created_at)
        now = int(now)
    except (TypeError, ValueError):
        return None
    if created <= 0:
        return None
    return (now - created) / 3600.0


def error_advice(code):
    """The documented meaning of one code. Pure. Empty for anything else."""
    return ADVICE.get(str(code or "").strip(), "")


def error_events(events):
    """Error-level messages in order. Pure. De-duplicated, never reordered."""
    out = []
    for item in events or []:
        if not isinstance(item, dict):
            continue
        if str(item.get("level") or "").lower() != "error":
            continue
        message = str(item.get("message") or "").strip()
        if message and message not in out:
            out.append(message)
    return out


def classify_job(row, now, stall_hours):
    """Grade one job. Pure. Returns (state, detail)."""
    row = row or {}
    status = str(row.get("status") or "")
    job_id = row.get("id") or "(no id)"
    if status == "failed" and row.get("code"):
        return ("job-failed",
                "%s: failed on %s with %s"
                % (job_id, row.get("param") or "an unnamed input",
                   row.get("code")))
    if status == "failed":
        return ("failed-without-error",
                "%s: failed with no error code on the job object, so the events "
                "feed is the only account of why" % job_id)
    if status == "validating_files":
        age = hours_since(row.get("created_at"), now)
        if age is not None and age >= stall_hours:
            return ("stalled-in-validation",
                    "%s: %.1f hours in validating_files, which is not progress"
                    % (job_id, age))
        return ("validating", "%s: validating files" % job_id)
    if status == "succeeded":
        return ("succeeded",
                "%s: succeeded, which is a different note" % job_id)
    if status == "cancelled":
        return ("cancelled", "%s: cancelled by somebody on purpose" % job_id)
    if status in ("queued", "running"):
        return ("running", "%s: %s" % (job_id, status))
    return ("unknown-status",
            "%s: status %r is not one this script recognises"
            % (job_id, status or "(none)"))


def repair_lines(state, code=""):
    """The repair for one verdict. Pure. Printed, never performed."""
    poll = ("poll the job to a terminal status in CI and fail the build on "
            "anything that is not succeeded. A 200 on create is a receipt, not "
            "a result.")
    if state == "job-failed":
        advice = error_advice(code)
        if advice:
            return [advice, poll]
        return ["the code %r is not one this script has a documented meaning "
                "for. Read error.message and the events feed above as printed, "
                "and do not act on a guess." % (code or "(none)"), poll]
    if state == "failed-without-error":
        return ["read GET /v1/fine_tuning/jobs/{id}/events for this job. The "
                "terminal status is all the job object recorded.", poll]
    if state == "stalled-in-validation":
        return ["read the events feed for the line that validation stopped on, "
                "and delete the file if it is a dead upload still counting "
                "against project storage.", poll]
    if state == "succeeded":
        return []
    return []


def fetch_jobs(key, timeout=30):
    """Paged GET of the job list. Returns (rows, error)."""
    rows = []
    params = {"limit": 100}
    headers = {"Authorization": "Bearer " + key}
    for _ in range(100):
        try:
            r = requests.get(JOBS_URL, headers=headers, params=params,
                             timeout=timeout)
        except requests.RequestException as exc:
            return (rows, "request failed: %s" % exc)
        if r.status_code != 200:
            return (rows, "HTTP %d %s" % (r.status_code, (r.text or "")[:160]))
        body = r.json()
        data = body.get("data") or []
        rows.extend(job_row(item) for item in data)
        if not body.get("has_more") or not data:
            break
        params["after"] = data[-1].get("id")
    return (rows, None)


def fetch_events(job_id, key, timeout=30):
    """GET the events feed for one job. Returns a list, empty on any problem."""
    try:
        r = requests.get("%s/%s/events" % (JOBS_URL, job_id),
                         headers={"Authorization": "Bearer " + key},
                         params={"limit": 100}, timeout=timeout)
    except requests.RequestException as exc:
        log.debug("events for %s failed: %s", job_id, exc)
        return []
    if r.status_code != 200:
        return []
    try:
        return list(reversed(r.json().get("data") or []))
    except ValueError:
        return []


def main():
    ap = argparse.ArgumentParser(description=__doc__)
    ap.add_argument("--stall-hours", type=float, default=2.0,
                    help="hours in validating_files that count as stalled")
    args = ap.parse_args()

    key = os.environ.get("OPENAI_API_KEY")
    if not key:
        log.error("set OPENAI_API_KEY to a project key set to Read Only. Both "
                  "calls are GETs of /v1/fine_tuning/jobs")
        return 2

    rows, err = fetch_jobs(key)
    if err:
        log.error("%s", err)
        return 2
    if not rows:
        log.info("no fine-tuning jobs in this project, so there is nothing to "
                 "grade")
        return 0

    now = int(time.time())
    findings = 0
    for row in sorted(rows, key=lambda r: -int(r.get("created_at") or 0)):
        state, detail = classify_job(row, now, args.stall_hours)
        emit = log.warning if state in FINDINGS else log.info
        emit("%-10s %-16s base %-16s created %s", row["id"], row["status"],
             row["model"] or "(none)",
             time.strftime("%Y-%m-%dT%H:%M:%SZ",
                           time.gmtime(row["created_at"] or 0)))
        if row.get("code"):
            emit("  error.code    %s", row["code"])
        if row.get("param"):
            emit("  error.param   %s", row["param"])
        if row.get("message"):
            emit("  error.message %s", row["message"])
        if state in ("job-failed", "failed-without-error",
                     "stalled-in-validation"):
            for message in error_events(fetch_events(row["id"], key))[:5]:
                emit("  event         %s", message)
        emit("%-21s %s", state, detail)
        for line in repair_lines(state, row.get("code")):
            emit("  repair: %s", line)
        if state in FINDINGS:
            findings += 1

    log.info("%d job(s), %d finding(s)", len(rows), findings)
    return 1 if findings else 0


if __name__ == "__main__":
    sys.exit(main())

/**
 * Find fine-tuning jobs that were accepted, then failed, and never read.
 *
 * Read only. GET /v1/fine_tuning/jobs, paginated, plus GET on the events feed
 * for jobs that failed. Nothing is created, cancelled or deleted.
 *
 * Creation is asynchronous, so the create call returning 200 says only that the
 * job was accepted. Validation and training failures surface on the job object
 * and nowhere else.
 *
 * The error codes are an open set: documented ones are translated into an
 * action, everything else is printed exactly as returned.
 */
const JOBS_URL = 'https://api.openai.com/v1/fine_tuning/jobs';

const ADVICE = {
  invalid_training_file:
    'the JSONL is malformed. One JSON object per line, no trailing blank line, '
    + 'no BOM, each row a messages array with at least one assistant turn, and '
    + 'one schema across every row.',
  invalid_validation_file:
    'the validation file has the same problem as a malformed training file, and '
    + 'error.param says which of the two was rejected.',
  invalid_n_examples:
    'the example count is out of range: too few rows to train on, or more than '
    + 'the method accepts. Count the lines before uploading.',
  exceeded_quota:
    'this is a billing problem rather than a data one. Editing the file will not '
    + "help; check the account's quota and spend limits.",
};

const FINDINGS = new Set(['job-failed', 'failed-without-error',
  'stalled-in-validation']);

/** One job, reduced. Pure. The error object is flattened. */
export function jobRow(body) {
  const job = (body && typeof body === 'object') ? body : {};
  const error = (job.error && typeof job.error === 'object') ? job.error : {};
  const created = Number(job.created_at ?? 0);
  return {
    id: String(job.id ?? ''),
    status: String(job.status ?? ''),
    model: String(job.model ?? ''),
    fine_tuned_model: String(job.fine_tuned_model ?? ''),
    created_at: Number.isFinite(created) ? Math.trunc(created) : 0,
    code: String(error.code ?? ''),
    param: String(error.param ?? ''),
    message: String(error.message ?? ''),
  };
}

/** Age in hours. Pure. The clock is an argument. */
export function hoursSince(createdAt, now) {
  const created = Number(createdAt);
  const at = Number(now);
  if (!Number.isFinite(created) || !Number.isFinite(at) || created <= 0) return null;
  return (at - created) / 3600;
}

/** The documented meaning of one code. Pure. Empty for anything else. */
export function errorAdvice(code) {
  return ADVICE[String(code ?? '').trim()] ?? '';
}

/** Error-level messages in order. Pure. De-duplicated, never reordered. */
export function errorEvents(events) {
  const out = [];
  for (const item of events ?? []) {
    if (!item || typeof item !== 'object') continue;
    if (String(item.level ?? '').toLowerCase() !== 'error') continue;
    const message = String(item.message ?? '').trim();
    if (message && !out.includes(message)) out.push(message);
  }
  return out;
}

/** Grade one job. Pure. Returns [state, detail]. */
export function classifyJob(row, now, stallHours) {
  const job = row ?? {};
  const status = String(job.status ?? '');
  const id = job.id || '(no id)';
  if (status === 'failed' && job.code) {
    return ['job-failed',
      `${id}: failed on ${job.param || 'an unnamed input'} with ${job.code}`];
  }
  if (status === 'failed') {
    return ['failed-without-error',
      `${id}: failed with no error code on the job object, so the events feed is `
      + 'the only account of why'];
  }
  if (status === 'validating_files') {
    const age = hoursSince(job.created_at, now);
    if (age !== null && age >= stallHours) {
      return ['stalled-in-validation',
        `${id}: ${age.toFixed(1)} hours in validating_files, which is not progress`];
    }
    return ['validating', `${id}: validating files`];
  }
  if (status === 'succeeded') {
    return ['succeeded', `${id}: succeeded, which is a different note`];
  }
  if (status === 'cancelled') {
    return ['cancelled', `${id}: cancelled by somebody on purpose`];
  }
  if (status === 'queued' || status === 'running') {
    return ['running', `${id}: ${status}`];
  }
  return ['unknown-status',
    `${id}: status '${status || '(none)'}' is not one this script recognises`];
}

/** The repair for one verdict. Pure. Printed, never performed. */
export function repairLines(state, code = '') {
  const poll = 'poll the job to a terminal status in CI and fail the build on '
    + 'anything that is not succeeded. A 200 on create is a receipt, not a result.';
  if (state === 'job-failed') {
    const advice = errorAdvice(code);
    if (advice) return [advice, poll];
    return [`the code '${code || '(none)'}' is not one this script has a `
      + 'documented meaning for. Read error.message and the events feed above as '
      + 'printed, and do not act on a guess.', poll];
  }
  if (state === 'failed-without-error') {
    return ['read GET /v1/fine_tuning/jobs/{id}/events for this job. The terminal '
      + 'status is all the job object recorded.', poll];
  }
  if (state === 'stalled-in-validation') {
    return ['read the events feed for the line that validation stopped on, and '
      + 'delete the file if it is a dead upload still counting against project '
      + 'storage.', poll];
  }
  return [];
}

async function fetchJobs(key) {
  const rows = [];
  const params = new URLSearchParams({ limit: '100' });
  for (let page = 0; page < 100; page += 1) {
    let res;
    try {
      res = await fetch(`${JOBS_URL}?${params.toString()}`,
        { headers: { Authorization: `Bearer ${key}` } });
    } catch (err) {
      return [rows, `request failed: ${err.message}`];
    }
    if (res.status !== 200) {
      return [rows, `HTTP ${res.status} ${(await res.text()).slice(0, 160)}`];
    }
    const body = await res.json();
    const data = body.data ?? [];
    for (const item of data) rows.push(jobRow(item));
    if (!body.has_more || !data.length) break;
    params.set('after', data[data.length - 1].id);
  }
  return [rows, null];
}

async function fetchEvents(jobId, key) {
  try {
    const res = await fetch(`${JOBS_URL}/${jobId}/events?limit=100`,
      { headers: { Authorization: `Bearer ${key}` } });
    if (res.status !== 200) return [];
    return [...((await res.json()).data ?? [])].reverse();
  } catch {
    return [];
  }
}

function args(argv) {
  const out = { stallHours: 2 };
  for (let i = 0; i < argv.length; i += 1) {
    if (argv[i] === '--stall-hours') out.stallHours = Number(argv[i += 1]);
  }
  return out;
}

async function main() {
  const opts = args(process.argv.slice(2));
  const key = process.env.OPENAI_API_KEY;
  if (!key) {
    console.error('set OPENAI_API_KEY to a project key set to Read Only. Both '
      + 'calls are GETs of /v1/fine_tuning/jobs');
    process.exitCode = 2;
    return;
  }
  const [rows, err] = await fetchJobs(key);
  if (err) {
    console.error(err);
    process.exitCode = 2;
    return;
  }
  if (!rows.length) {
    console.log('no fine-tuning jobs in this project, so there is nothing to grade');
    return;
  }

  const now = Math.trunc(Date.now() / 1000);
  let findings = 0;
  for (const row of [...rows].sort((a, b) => b.created_at - a.created_at)) {
    const [state, detail] = classifyJob(row, now, opts.stallHours);
    const when = row.created_at
      ? `${new Date(row.created_at * 1000).toISOString().slice(0, 19)}Z` : '(unknown)';
    console.log(`${row.id.padEnd(10)} ${row.status.padEnd(16)} base `
      + `${(row.model || '(none)').padEnd(16)} created ${when}`);
    if (row.code) console.log(`  error.code    ${row.code}`);
    if (row.param) console.log(`  error.param   ${row.param}`);
    if (row.message) console.log(`  error.message ${row.message}`);
    if (FINDINGS.has(state)) {
      const events = errorEvents(await fetchEvents(row.id, key)).slice(0, 5);
      for (const message of events) console.log(`  event         ${message}`);
    }
    console.log(`${state.padEnd(21)} ${detail}`);
    for (const line of repairLines(state, row.code)) console.log(`  repair: ${line}`);
    if (FINDINGS.has(state)) findings += 1;
  }
  console.log(`${rows.length} job(s), ${findings} finding(s)`);
  process.exitCode = findings ? 1 : 0;
}

if (import.meta.url === `file://${process.argv[1]}`) await main();

from openai_fine_tune_failures import (classify_job, error_advice,
                                      error_events, hours_since, job_row,
                                      repair_lines)

NOW = 1_800_000_000
HOUR = 3600


def job(status, code="", param="", message="", hours_old=1.0, jid="ftjob_a1"):
    return job_row({"id": jid, "object": "fine_tuning.job", "status": status,
                    "model": "gpt-5.6-terra",
                    "created_at": NOW - int(hours_old * HOUR),
                    "fine_tuned_model": None, "trained_tokens": None,
                    "error": ({"code": code, "message": message,
                               "param": param} if code or message else None)})


def test_a_failed_job_names_the_rejected_input_and_the_documented_fix():
    row = job("failed", "invalid_training_file", "training_file",
              "The job failed due to an invalid training file.")
    state, detail = classify_job(row, NOW, 2.0)
    assert state == "job-failed"
    assert "failed on training_file with invalid_training_file" in detail
    lines = repair_lines(state, row["code"])
    assert lines[0] == error_advice("invalid_training_file")
    assert "no trailing blank line" in lines[0]
    assert any("receipt, not a result" in line for line in lines)


def test_an_unknown_code_is_printed_and_never_interpreted():
    row = job("failed", "some_new_code_2027", "training_file", "...")
    state, _ = classify_job(row, NOW, 2.0)
    assert state == "job-failed"
    assert error_advice("some_new_code_2027") == ""
    lines = repair_lines(state, row["code"])
    assert "some_new_code_2027" in lines[0]
    assert "do not act on a guess" in lines[0]
    # exceeded_quota is documented, and it is not a data problem.
    assert "billing problem" in error_advice("exceeded_quota")
    assert "Editing the file will not help" in error_advice("exceeded_quota")


def test_hours_in_validating_files_is_its_own_finding():
    stalled = job("validating_files", hours_old=9.4, jid="ftjob_b2")
    state, detail = classify_job(stalled, NOW, 2.0)
    assert state == "stalled-in-validation"
    assert "9.4 hours in validating_files" in detail
    assert any("dead upload" in line for line in repair_lines(state))
    # The same job an hour in is simply validating.
    fresh = job("validating_files", hours_old=1.0, jid="ftjob_b3")
    assert classify_job(fresh, NOW, 2.0)[0] == "validating"
    assert abs(hours_since(NOW - 5 * HOUR, NOW) - 5.0) < 1e-9
    assert hours_since(0, NOW) is None


def test_a_failure_with_no_error_object_is_sent_to_the_events_feed():
    row = job("failed")
    assert row["code"] == "" and row["param"] == ""
    state, detail = classify_job(row, NOW, 2.0)
    assert state == "failed-without-error"
    assert "the only account of why" in detail
    assert any("/events" in line for line in repair_lines(state))


def test_a_succeeded_job_is_handed_to_the_other_note():
    state, detail = classify_job(job("succeeded", hours_old=200.0), NOW, 2.0)
    assert state == "succeeded"
    assert "a different note" in detail
    assert repair_lines(state) == []
    assert classify_job(job("cancelled"), NOW, 2.0)[0] == "cancelled"
    assert classify_job(job("running"), NOW, 2.0)[0] == "running"
    assert classify_job(job("beaming_up"), NOW, 2.0)[0] == "unknown-status"


def test_the_events_feed_is_filtered_to_errors_and_kept_in_order():
    feed = [{"level": "info", "message": "Created fine-tuning job"},
            {"level": "error", "message": "line 4108 has no assistant message"},
            {"level": "warn", "message": "..."},
            {"level": "ERROR", "message": "line 4108 has no assistant message"},
            {"level": "error", "message": "validation failed"},
            "not a dict"]
    assert error_events(feed) == ["line 4108 has no assistant message",
                                  "validation failed"]
    assert error_events(None) == []
    assert job_row(None)["id"] == ""
    assert job_row({"created_at": "nonsense"})["created_at"] == 0

import { test } from 'node:test';
import assert from 'node:assert/strict';
import { classifyJob, errorAdvice, errorEvents, hoursSince, jobRow,
         repairLines } from './openai-fine-tune-failures.mjs';

const NOW = 1_800_000_000;
const HOUR = 3600;

const job = (status, { code = '', param = '', message = '', hoursOld = 1,
                       id = 'ftjob_a1' } = {}) => jobRow({
  id,
  object: 'fine_tuning.job',
  status,
  model: 'gpt-5.6-terra',
  created_at: NOW - Math.trunc(hoursOld * HOUR),
  fine_tuned_model: null,
  trained_tokens: null,
  error: (code || message) ? { code, message, param } : null,
});

test('a failed job names the rejected input and the documented fix', () => {
  const row = job('failed', { code: 'invalid_training_file', param: 'training_file',
                              message: 'The job failed due to an invalid training file.' });
  const [state, detail] = classifyJob(row, NOW, 2);
  assert.equal(state, 'job-failed');
  assert.ok(detail.includes('failed on training_file with invalid_training_file'));
  const lines = repairLines(state, row.code);
  assert.equal(lines[0], errorAdvice('invalid_training_file'));
  assert.ok(lines[0].includes('no trailing blank line'));
  assert.ok(lines.some((l) => l.includes('receipt, not a result')));
});

test('an unknown code is printed and never interpreted', () => {
  const row = job('failed', { code: 'some_new_code_2027', param: 'training_file',
                              message: '...' });
  assert.equal(classifyJob(row, NOW, 2)[0], 'job-failed');
  assert.equal(errorAdvice('some_new_code_2027'), '');
  const lines = repairLines('job-failed', row.code);
  assert.ok(lines[0].includes('some_new_code_2027'));
  assert.ok(lines[0].includes('do not act on a guess'));
  assert.ok(errorAdvice('exceeded_quota').includes('billing problem'));
  assert.ok(errorAdvice('exceeded_quota').includes('Editing the file will not help'));
});

test('hours in validating_files is its own finding', () => {
  const stalled = job('validating_files', { hoursOld: 9.4, id: 'ftjob_b2' });
  const [state, detail] = classifyJob(stalled, NOW, 2);
  assert.equal(state, 'stalled-in-validation');
  assert.ok(detail.includes('9.4 hours in validating_files'));
  assert.ok(repairLines(state).some((l) => l.includes('dead upload')));
  assert.equal(classifyJob(job('validating_files', { hoursOld: 1 }), NOW, 2)[0],
               'validating');
  assert.ok(Math.abs(hoursSince(NOW - 5 * HOUR, NOW) - 5) < 1e-9);
  assert.equal(hoursSince(0, NOW), null);
});

test('a failure with no error object is sent to the events feed', () => {
  const row = job('failed');
  assert.equal(row.code, '');
  assert.equal(row.param, '');
  const [state, detail] = classifyJob(row, NOW, 2);
  assert.equal(state, 'failed-without-error');
  assert.ok(detail.includes('the only account of why'));
  assert.ok(repairLines(state).some((l) => l.includes('/events')));
});

test('a succeeded job is handed to the other note', () => {
  const [state, detail] = classifyJob(job('succeeded', { hoursOld: 200 }), NOW, 2);
  assert.equal(state, 'succeeded');
  assert.ok(detail.includes('a different note'));
  assert.deepEqual(repairLines(state), []);
  assert.equal(classifyJob(job('cancelled'), NOW, 2)[0], 'cancelled');
  assert.equal(classifyJob(job('running'), NOW, 2)[0], 'running');
  assert.equal(classifyJob(job('beaming_up'), NOW, 2)[0], 'unknown-status');
});

test('the events feed is filtered to errors and kept in order', () => {
  const feed = [{ level: 'info', message: 'Created fine-tuning job' },
                { level: 'error', message: 'line 4108 has no assistant message' },
                { level: 'warn', message: '...' },
                { level: 'ERROR', message: 'line 4108 has no assistant message' },
                { level: 'error', message: 'validation failed' },
                'not a dict'];
  assert.deepEqual(errorEvents(feed),
                   ['line 4108 has no assistant message', 'validation failed']);
  assert.deepEqual(errorEvents(null), []);
  assert.equal(jobRow(null).id, '');
  assert.equal(jobRow({ created_at: 'nonsense' }).created_at, 0);
});