A project or workspace ceiling set below the org limit

"""Find a container whose rate limit was set below the organization's.

Read only. Every request is a GET, on either or both providers:

  Anthropic, Admin API key
    GET /v1/organizations/rate_limits
    GET /v1/organizations/workspaces
    GET /v1/organizations/workspaces/{workspace_id}/rate_limits
  OpenAI, admin key
    GET /v1/organization/projects
    GET /v1/organization/projects/{project_id}/rate_limits

Nothing here reads a response header and nothing here reads traffic. The subject
is the configured ceiling on a container, which is legible whether or not that
container has sent a single request this month.

Anthropic returns each workspace override with org_limit beside value on the
same object, so the comparison is exact. OpenAI's project.rate_limit object
carries no organization value at all, so the peer maximum across projects stands
in for the tier and is reported as the proxy it is.

The repair is a write on both providers. It is printed, never performed.
"""
import argparse
import logging
import os
import sys

import requests

logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s")
log = logging.getLogger("rate_limit_below_org_audit")

ANTHROPIC = "https://api.anthropic.com/v1"
OPENAI = "https://api.openai.com/v1"
ANTHROPIC_VERSION = "2023-06-01"

# The three limiters a model group carries. Other group types carry their own
# types (enqueued_batch_requests and so on), which is why nothing here assumes
# this tuple is the whole vocabulary: it is only used to order output.
LIMITER_ORDER = ("requests_per_minute", "input_tokens_per_minute",
                 "output_tokens_per_minute")

# Severity order, worst first. verdict() walks this, so adding a state means
# deciding where it sits rather than hoping the dict happened to be ordered.
SEVERITY = ("throttled-below-org", "override-pinned-at-org", "override-above-org",
            "limiter-inherited", "org-limit-unknown", "override-in-range",
            "no-override")

FINDINGS = ("throttled-below-org", "override-pinned-at-org", "override-above-org",
            "project-outlier")


def num(value):
    """An int, or None. Pure.

    None is a real answer here and must survive: org_limit is documented as
    nullable, and coercing a null to 0 would turn "cannot be graded" into
    "throttled to nothing", which is the loudest possible wrong answer.
    """
    if value is None or isinstance(value, bool):
        return None
    try:
        return int(value)
    except (TypeError, ValueError):
        return None


def group_label(entry):
    """A stable printable name for one rate limit group. Pure.

    The organization endpoint and the workspace endpoint describe the same
    groups with the same models list, so labelling them identically is what lets
    the two be joined without matching on model strings by hand.
    """
    entry = entry or {}
    gtype = str(entry.get("group_type") or "").strip() or "unknown_group"
    models = sorted(str(m) for m in (entry.get("models") or []) if m)
    if not models:
        return gtype
    extra = len(models) - 1
    return "%s:%s%s" % (gtype, models[0], (" +%d" % extra) if extra else "")


def limits_of(entry):
    """{limiter_type: value} for one group entry. Pure. Unparseable values drop."""
    out = {}
    for row in ((entry or {}).get("limits") or []):
        row = row or {}
        ltype = str(row.get("type") or "").strip()
        value = num(row.get("value"))
        if ltype and value is not None:
            out[ltype] = value
    return out


def org_index(pages):
    """{group_label: {limiter_type: value}} from the organization endpoint. Pure.

    Takes an iterable of page payloads so a paginated read folds without the
    caller flattening first.
    """
    out = {}
    for page in pages or []:
        for entry in ((page or {}).get("data") or []):
            out.setdefault(group_label(entry), {}).update(limits_of(entry))
    return out


def overrides_of(entry):
    """[(limiter_type, value, org_limit)] for one workspace group. Pure.

    org_limit stays None when the API reports null. The caller decides whether
    to fall back to the organization listing; this function does not guess.
    """
    out = []
    for row in ((entry or {}).get("limits") or []):
        row = row or {}
        ltype = str(row.get("type") or "").strip()
        if not ltype:
            continue
        out.append((ltype, num(row.get("value")), num(row.get("org_limit"))))
    out.sort(key=lambda r: (LIMITER_ORDER.index(r[0])
                            if r[0] in LIMITER_ORDER else len(LIMITER_ORDER), r[0]))
    return out


def grade_override(value, org_limit, floor=0.5):
    """Grade one workspace limiter against the organization value. Pure.

    Returns (state, detail). The equality case is deliberately not folded into
    the ratio: an override equal to today's organization value is a pin, and a
    threshold check that passes everything at 1.0 will never say so.
    """
    if value is None:
        return ("no-override", "inherits the organization value")
    if org_limit is None:
        return ("org-limit-unknown",
                "value is %s and the organization publishes no number for this "
                "limiter, so the override cannot be graded" % fmt(value))
    if value <= 0:
        return ("throttled-below-org",
                "set to %s, which stops this limiter in this container entirely"
                % fmt(value))
    if value > org_limit:
        return ("override-above-org",
                "%s is above the organization's %s, and the organization limit "
                "applies anyway" % (fmt(value), fmt(org_limit)))
    if value == org_limit:
        return ("override-pinned-at-org",
                "%s, equal to the organization value today, so it will not "
                "follow the next increase" % fmt(value))
    share = float(value) / float(org_limit)
    if share <= floor:
        return ("throttled-below-org",
                "%s of %s (%.0f%%)" % (fmt(value), fmt(org_limit), share * 100))
    return ("override-in-range",
            "%s of %s (%.0f%%)" % (fmt(value), fmt(org_limit), share * 100))


def inherited_limiters(entry, org_types):
    """Limiter types the organization publishes that this group did not override.

    Pure. Returns [(limiter_type, org_value)] in a stable order.
    """
    overridden = {t for t, value, _ in overrides_of(entry) if value is not None}
    rows = [(t, v) for t, v in (org_types or {}).items() if t not in overridden]
    rows.sort(key=lambda r: (LIMITER_ORDER.index(r[0])
                             if r[0] in LIMITER_ORDER else len(LIMITER_ORDER), r[0]))
    return rows


def verdict(states):
    """Roll one container's limiter states into a single word. Pure."""
    present = set(states or [])
    for state in SEVERITY:
        if state in present:
            return state
    return "no-override"


def openai_matrix(by_project):
    """{model: {project_id: {"rpm": int|None, "tpm": int|None}}}. Pure.

    Rows with no model string are dropped rather than collected under a blank
    key, because a blank key would then be compared against real ones.
    """
    out = {}
    for pid, rows in sorted((by_project or {}).items()):
        for row in rows or []:
            row = row or {}
            model = str(row.get("model") or "").strip()
            if not model:
                continue
            out.setdefault(model, {})[str(pid)] = {
                "rpm": num(row.get("max_requests_per_1_minute")),
                "tpm": num(row.get("max_tokens_per_1_minute")),
            }
    return out


def openai_outliers(matrix, floor=0.5):
    """[(model, project_id, dimension, value, peer_max)]. Pure. Worst first.

    A model row carried by fewer than two projects is skipped: with one project
    there is no peer to compare against, and the object carries no organization
    value to compare against instead.
    """
    out = []
    for model, projects in sorted((matrix or {}).items()):
        if len(projects) < 2:
            continue
        for dim in ("rpm", "tpm"):
            values = {p: (v or {}).get(dim) for p, v in projects.items()}
            usable = {p: v for p, v in values.items() if v is not None and v > 0}
            if len(usable) < 2:
                continue
            peer_max = max(usable.values())
            for pid, value in sorted(usable.items()):
                if value <= peer_max * floor:
                    out.append((model, pid, dim, value, peer_max))
    out.sort(key=lambda r: (r[3] / float(r[4]), r[0], r[1], r[2]))
    return out


def fmt(value):
    """Thousands separators, or a dash for None. Pure."""
    if value is None:
        return "-"
    return "{:,}".format(int(value))


def repair_lines(state):
    """The repair for one state. Pure. Printed, never performed."""
    if state == "throttled-below-org":
        return ["this container is capped well under the organization ceiling. "
                "On Anthropic open the workspace in the Console, Rate limits "
                "tab, and raise or remove the override; there is no write "
                "endpoint for it.",
                "check the container id against what production actually uses "
                "before raising anything. A staging id that followed the code "
                "into production is repaired by changing the id, not the limit."]
    if state == "override-pinned-at-org":
        return ["an override equal to today's organization value is a pin, not "
                "a no-op. Delete the override so the container follows the next "
                "tier increase instead of staying on this number.",
                "if the equality is deliberate, write it down somewhere the "
                "next tier increase will be read, because nothing in the API "
                "will mention it again."]
    if state == "override-above-org":
        return ["an override above the organization value has no effect: "
                "organization limits always apply. Remove it so the "
                "configuration says what is actually enforced."]
    if state == "project-outlier":
        return ["raise it with the admin update call at "
                "/v1/organization/projects/{project_id}/rate_limits/"
                "{rate_limit_id}, sending the dimension you want changed. That "
                "is a write and this script does not make it.",
                "the peer maximum is a proxy for the tier value, not the tier "
                "value: this object carries no organization number. Confirm "
                "against the tier before treating the gap as the whole story."]
    if state == "org-limit-unknown":
        return ["the organization publishes no number for this limiter, so the "
                "override is unjudgeable rather than fine. Read "
                "/v1/organizations/rate_limits for the group before acting."]
    return []


def get(session, url, headers=None, **params):
    r = session.get(url, params=params, headers=headers or {}, timeout=90)
    if r.status_code in (401, 403):
        raise SystemExit("%d from %s: this path needs an organization scoped "
                         "read credential, not a workspace or project key"
                         % (r.status_code, url))
    r.raise_for_status()
    return r.json()


def anthropic_pages(session, path, **params):
    """Walk an Anthropic next_page cursor listing, yielding whole payloads."""
    params = dict(params)
    for _ in range(50):
        page = get(session, ANTHROPIC + path, **params)
        yield page
        nxt = page.get("next_page")
        if not nxt:
            return
        params["page"] = nxt


def anthropic_cursor(session, path, **params):
    """Walk an Anthropic after_id listing, yielding items."""
    params = dict(params)
    for _ in range(50):
        page = get(session, ANTHROPIC + path, **params)
        data = page.get("data") or []
        for item in data:
            yield item
        if not page.get("has_more") or not data:
            return
        params["after_id"] = page.get("last_id") or (data[-1] or {}).get("id")


def openai_cursor(session, path, **params):
    """Walk an OpenAI after/last_id listing, yielding items."""
    params = dict(params)
    for _ in range(50):
        page = get(session, OPENAI + path, **params)
        data = page.get("data") or []
        for item in data:
            yield item
        if not page.get("has_more") or not data:
            return
        params["after"] = page.get("last_id") or (data[-1] or {}).get("id")


def audit_anthropic(key, floor):
    s = requests.Session()
    s.headers.update({"x-api-key": key,
                      "anthropic-version": ANTHROPIC_VERSION,
                      "User-Agent": "rate-limit-below-org-audit/1.0"})
    org = org_index(anthropic_pages(s, "/organizations/rate_limits"))
    spaces = list(anthropic_cursor(s, "/organizations/workspaces", limit=100))
    log.info("anthropic: %d workspace(s), %d organization rate limit group(s)",
             len(spaces), len(org))

    findings = 0
    for space in spaces:
        wid = (space or {}).get("id") or "?"
        name = (space or {}).get("name") or "(unnamed)"
        entries = []
        for page in anthropic_pages(
                s, "/organizations/workspaces/%s/rate_limits" % wid):
            entries.extend(page.get("data") or [])
        if not entries:
            log.info("%-22s %s %s: inherits every organization limit",
                     "no-override", wid, name)
            continue
        for entry in entries:
            label = group_label(entry)
            org_types = org.get(label) or {}
            states = []
            rows = []
            for ltype, value, org_limit in overrides_of(entry):
                fallback = org_limit if org_limit is not None else org_types.get(ltype)
                state, detail = grade_override(value, fallback, floor)
                states.append(state)
                rows.append((ltype, state, detail))
            inherited = inherited_limiters(entry, org_types)
            if inherited and states:
                states.append("limiter-inherited")
            state = verdict(states)
            emit = log.warning if state in FINDINGS else log.info
            emit("%-22s %s %s / %s", state, wid, name, label)
            for ltype, row_state, detail in rows:
                emit("  %-26s %s", ltype, detail)
            for ltype, value in inherited:
                emit("  inherited: %s (%s from the organization)", ltype, fmt(value))
            for line in repair_lines(state):
                emit("  repair: %s", line)
            if state in FINDINGS:
                findings += 1
    return findings


def audit_openai(key, floor):
    s = requests.Session()
    s.headers.update({"Authorization": "Bearer " + key,
                      "User-Agent": "rate-limit-below-org-audit/1.0"})
    projects = list(openai_cursor(s, "/organization/projects", limit=100,
                                  include_archived="false"))
    by_project = {}
    names = {}
    for project in projects:
        pid = (project or {}).get("id") or "?"
        names[pid] = (project or {}).get("name") or "(unnamed)"
        by_project[pid] = list(openai_cursor(
            s, "/organization/projects/%s/rate_limits" % pid, limit=100))

    matrix = openai_matrix(by_project)
    comparable = sum(1 for m in matrix.values() if len(m) >= 2)
    log.info("openai: %d project(s), %d model row(s) carried by 2 or more "
             "projects", len(projects), comparable)
    if len(projects) < 2:
        log.info("%-22s one project only: this object carries no organization "
                 "value, so there is nothing to compare against", "no-peer")
        return 0

    rows = openai_outliers(matrix, floor)
    dimension = {"rpm": "max_requests_per_1_minute",
                 "tpm": "max_tokens_per_1_minute"}
    seen = set()
    for model, pid, dim, value, peer_max in rows:
        log.warning("%-22s %s %s  %s", "project-outlier", pid,
                    names.get(pid, "(unnamed)"), model)
        log.warning("  %-26s %s against a peer maximum of %s (%.0f%%)",
                    dimension[dim], fmt(value), fmt(peer_max),
                    100.0 * value / peer_max)
        seen.add((pid, model))
    if rows:
        for line in repair_lines("project-outlier"):
            log.warning("  repair: %s", line)
    return len(seen)


def main():
    ap = argparse.ArgumentParser(description=__doc__)
    ap.add_argument("--floor", type=float, default=0.5,
                    help="report an override at or below this share of the "
                         "organization value (default 0.5)")
    args = ap.parse_args()

    anthropic_key = os.environ.get("ANTHROPIC_ADMIN_KEY")
    openai_key = os.environ.get("OPENAI_ADMIN_KEY")
    if not anthropic_key and not openai_key:
        log.error("set ANTHROPIC_ADMIN_KEY, OPENAI_ADMIN_KEY, or both. Each "
                  "must be an organization scoped read credential; a workspace "
                  "or project key cannot reach these paths")
        return 2

    findings = 0
    if anthropic_key:
        findings += audit_anthropic(anthropic_key, args.floor)
    if openai_key:
        findings += audit_openai(openai_key, args.floor)

    log.info("%d finding(s)", findings)
    return 1 if findings else 0


if __name__ == "__main__":
    sys.exit(main())

/**
 * Find a container whose rate limit was set below the organization's.
 *
 * Read only. Every request is a GET, on either or both providers. Nothing here
 * reads a response header and nothing here reads traffic: the subject is the
 * configured ceiling on a container, legible whether or not that container has
 * sent a single request this month.
 *
 * Anthropic returns each workspace override with org_limit beside value on the
 * same object. OpenAI's project.rate_limit object carries no organization value
 * at all, so the peer maximum across projects stands in for the tier and is
 * reported as the proxy it is. The repair is a write and is printed only.
 */
const ANTHROPIC = 'https://api.anthropic.com/v1';
const OPENAI = 'https://api.openai.com/v1';
const ANTHROPIC_VERSION = '2023-06-01';

export const LIMITER_ORDER = ['requests_per_minute', 'input_tokens_per_minute',
                              'output_tokens_per_minute'];

const SEVERITY = ['throttled-below-org', 'override-pinned-at-org', 'override-above-org',
                  'limiter-inherited', 'org-limit-unknown', 'override-in-range',
                  'no-override'];

const FINDINGS = new Set(['throttled-below-org', 'override-pinned-at-org',
                          'override-above-org', 'project-outlier']);

/** An integer, or null. Pure. null is a real answer and must survive. */
export function num(value) {
  if (value === null || value === undefined || typeof value === 'boolean') return null;
  const n = Number(value);
  return Number.isFinite(n) ? Math.trunc(n) : null;
}

/** A stable printable name for one rate limit group. Pure. */
export function groupLabel(entry) {
  const gtype = String(entry?.group_type ?? '').trim() || 'unknown_group';
  const models = (entry?.models ?? []).filter(Boolean).map(String).sort();
  if (models.length === 0) return gtype;
  const extra = models.length - 1;
  return `${gtype}:${models[0]}${extra ? ` +${extra}` : ''}`;
}

/** {limiterType: value} for one group entry. Pure. */
export function limitsOf(entry) {
  const out = {};
  for (const row of entry?.limits ?? []) {
    const ltype = String(row?.type ?? '').trim();
    const value = num(row?.value);
    if (ltype && value !== null) out[ltype] = value;
  }
  return out;
}

/** {groupLabel: {limiterType: value}} from the organization endpoint. Pure. */
export function orgIndex(pages) {
  const out = {};
  for (const page of pages ?? []) {
    for (const entry of page?.data ?? []) {
      out[groupLabel(entry)] = { ...(out[groupLabel(entry)] ?? {}), ...limitsOf(entry) };
    }
  }
  return out;
}

const rank = (t) => (LIMITER_ORDER.indexOf(t) === -1 ? LIMITER_ORDER.length
                                                     : LIMITER_ORDER.indexOf(t));

/** [[limiterType, value, orgLimit]] for one workspace group. Pure. */
export function overridesOf(entry) {
  const out = [];
  for (const row of entry?.limits ?? []) {
    const ltype = String(row?.type ?? '').trim();
    if (!ltype) continue;
    out.push([ltype, num(row?.value), num(row?.org_limit)]);
  }
  out.sort((a, b) => (rank(a[0]) - rank(b[0])) || a[0].localeCompare(b[0]));
  return out;
}

/** Thousands separators, or a dash for null. Pure. */
export function fmt(value) {
  if (value === null || value === undefined) return '-';
  return Math.trunc(Number(value)).toLocaleString('en-US');
}

/** Grade one workspace limiter against the organization value. Pure. */
export function gradeOverride(value, orgLimit, floor = 0.5) {
  if (value === null || value === undefined) {
    return ['no-override', 'inherits the organization value'];
  }
  if (orgLimit === null || orgLimit === undefined) {
    return ['org-limit-unknown',
            `value is ${fmt(value)} and the organization publishes no number for `
            + 'this limiter, so the override cannot be graded'];
  }
  if (value <= 0) {
    return ['throttled-below-org',
            `set to ${fmt(value)}, which stops this limiter in this container entirely`];
  }
  if (value > orgLimit) {
    return ['override-above-org',
            `${fmt(value)} is above the organization's ${fmt(orgLimit)}, and the `
            + 'organization limit applies anyway'];
  }
  if (value === orgLimit) {
    return ['override-pinned-at-org',
            `${fmt(value)}, equal to the organization value today, so it will not `
            + 'follow the next increase'];
  }
  const share = value / orgLimit;
  const detail = `${fmt(value)} of ${fmt(orgLimit)} (${Math.round(share * 100)}%)`;
  return [share <= floor ? 'throttled-below-org' : 'override-in-range', detail];
}

/** [[limiterType, orgValue]] the organization publishes and the group did not override. */
export function inheritedLimiters(entry, orgTypes) {
  const overridden = new Set(overridesOf(entry).filter((r) => r[1] !== null)
    .map((r) => r[0]));
  const rows = Object.entries(orgTypes ?? {}).filter(([t]) => !overridden.has(t));
  rows.sort((a, b) => (rank(a[0]) - rank(b[0])) || a[0].localeCompare(b[0]));
  return rows;
}

/** Roll one container's limiter states into a single word. Pure. */
export function verdict(states) {
  const present = new Set(states ?? []);
  for (const state of SEVERITY) if (present.has(state)) return state;
  return 'no-override';
}

/** {model: {projectId: {rpm, tpm}}}. Pure. */
export function openaiMatrix(byProject) {
  const out = {};
  for (const pid of Object.keys(byProject ?? {}).sort()) {
    for (const row of byProject[pid] ?? []) {
      const model = String(row?.model ?? '').trim();
      if (!model) continue;
      (out[model] ??= {})[String(pid)] = {
        rpm: num(row?.max_requests_per_1_minute),
        tpm: num(row?.max_tokens_per_1_minute),
      };
    }
  }
  return out;
}

/** [[model, projectId, dimension, value, peerMax]]. Pure. Worst first. */
export function openaiOutliers(matrix, floor = 0.5) {
  const out = [];
  for (const model of Object.keys(matrix ?? {}).sort()) {
    const projects = matrix[model];
    if (Object.keys(projects).length < 2) continue;
    for (const dim of ['rpm', 'tpm']) {
      const usable = Object.entries(projects)
        .map(([p, v]) => [p, v?.[dim]])
        .filter(([, v]) => v !== null && v !== undefined && v > 0);
      if (usable.length < 2) continue;
      const peerMax = Math.max(...usable.map(([, v]) => v));
      for (const [pid, value] of usable.sort((a, b) => a[0].localeCompare(b[0]))) {
        if (value <= peerMax * floor) out.push([model, pid, dim, value, peerMax]);
      }
    }
  }
  out.sort((a, b) => (a[3] / a[4]) - (b[3] / b[4])
    || a[0].localeCompare(b[0]) || a[1].localeCompare(b[1]));
  return out;
}

/** The repair for one state. Pure. Printed, never performed. */
export function repairLines(state) {
  if (state === 'throttled-below-org') {
    return ['this container is capped well under the organization ceiling. On '
      + 'Anthropic open the workspace in the Console, Rate limits tab, and raise '
      + 'or remove the override; there is no write endpoint for it.',
      'check the container id against what production actually uses before '
      + 'raising anything. A staging id that followed the code into production '
      + 'is repaired by changing the id, not the limit.'];
  }
  if (state === 'override-pinned-at-org') {
    return ['an override equal to today\'s organization value is a pin, not a '
      + 'no-op. Delete the override so the container follows the next tier '
      + 'increase instead of staying on this number.',
      'if the equality is deliberate, write it down somewhere the next tier '
      + 'increase will be read, because nothing in the API will mention it again.'];
  }
  if (state === 'override-above-org') {
    return ['an override above the organization value has no effect: organization '
      + 'limits always apply. Remove it so the configuration says what is '
      + 'actually enforced.'];
  }
  if (state === 'project-outlier') {
    return ['raise it with the admin update call at /v1/organization/projects/'
      + '{project_id}/rate_limits/{rate_limit_id}, sending the dimension you want '
      + 'changed. That is a write and this script does not make it.',
      'the peer maximum is a proxy for the tier value, not the tier value: this '
      + 'object carries no organization number. Confirm against the tier before '
      + 'treating the gap as the whole story.'];
  }
  if (state === 'org-limit-unknown') {
    return ['the organization publishes no number for this limiter, so the '
      + 'override is unjudgeable rather than fine. Read '
      + '/v1/organizations/rate_limits for the group before acting.'];
  }
  return [];
}

async function read(url, headers, params) {
  const target = new URL(url);
  for (const [k, v] of Object.entries(params ?? {})) target.searchParams.set(k, String(v));
  const r = await fetch(target, { headers });
  if (r.status === 401 || r.status === 403) {
    throw new Error(`${r.status} from ${url}: this path needs an organization `
                    + 'scoped read credential, not a workspace or project key');
  }
  if (!r.ok) throw new Error(`${r.status} ${await r.text()}`);
  return r.json();
}

async function anthropicPages(headers, path, params) {
  const out = [];
  const q = { ...(params ?? {}) };
  for (let i = 0; i < 50; i += 1) {
    const page = await read(ANTHROPIC + path, headers, q);
    out.push(page);
    if (!page.next_page) break;
    q.page = page.next_page;
  }
  return out;
}

async function cursor(base, headers, path, params, cursorKey) {
  const out = [];
  const q = { ...(params ?? {}) };
  for (let i = 0; i < 50; i += 1) {
    const page = await read(base + path, headers, q);
    const data = page.data ?? [];
    out.push(...data);
    if (!page.has_more || data.length === 0) break;
    q[cursorKey] = page.last_id ?? data[data.length - 1]?.id;
  }
  return out;
}

async function auditAnthropic(key, floor) {
  const headers = { 'x-api-key': key, 'anthropic-version': ANTHROPIC_VERSION,
                    'User-Agent': 'rate-limit-below-org-audit/1.0' };
  const org = orgIndex(await anthropicPages(headers, '/organizations/rate_limits'));
  const spaces = await cursor(ANTHROPIC, headers, '/organizations/workspaces',
                              { limit: 100 }, 'after_id');
  console.log(`anthropic: ${spaces.length} workspace(s), ${Object.keys(org).length} `
              + 'organization rate limit group(s)');

  let findings = 0;
  for (const space of spaces) {
    const wid = space?.id ?? '?';
    const name = space?.name ?? '(unnamed)';
    const pages = await anthropicPages(
      headers, `/organizations/workspaces/${wid}/rate_limits`);
    const entries = pages.flatMap((p) => p.data ?? []);
    if (entries.length === 0) {
      console.log(`${'no-override'.padEnd(22)} ${wid} ${name}: inherits every `
                  + 'organization limit');
      continue;
    }
    for (const entry of entries) {
      const label = groupLabel(entry);
      const orgTypes = org[label] ?? {};
      const states = [];
      const rows = [];
      for (const [ltype, value, orgLimit] of overridesOf(entry)) {
        const fallback = orgLimit === null ? (orgTypes[ltype] ?? null) : orgLimit;
        const [state, detail] = gradeOverride(value, fallback, floor);
        states.push(state);
        rows.push([ltype, detail]);
      }
      const inherited = inheritedLimiters(entry, orgTypes);
      if (inherited.length && states.length) states.push('limiter-inherited');
      const state = verdict(states);
      console.log(`${state.padEnd(22)} ${wid} ${name} / ${label}`);
      for (const [ltype, detail] of rows) console.log(`  ${ltype.padEnd(26)} ${detail}`);
      for (const [ltype, value] of inherited) {
        console.log(`  inherited: ${ltype} (${fmt(value)} from the organization)`);
      }
      for (const line of repairLines(state)) console.log(`  repair: ${line}`);
      if (FINDINGS.has(state)) findings += 1;
    }
  }
  return findings;
}

async function auditOpenai(key, floor) {
  const headers = { Authorization: `Bearer ${key}`,
                    'User-Agent': 'rate-limit-below-org-audit/1.0' };
  const projects = await cursor(OPENAI, headers, '/organization/projects',
                                { limit: 100, include_archived: 'false' }, 'after');
  const byProject = {};
  const names = {};
  for (const project of projects) {
    const pid = project?.id ?? '?';
    names[pid] = project?.name ?? '(unnamed)';
    byProject[pid] = await cursor(
      OPENAI, headers, `/organization/projects/${pid}/rate_limits`, { limit: 100 }, 'after');
  }

  const matrix = openaiMatrix(byProject);
  const comparable = Object.values(matrix).filter((m) => Object.keys(m).length >= 2).length;
  console.log(`openai: ${projects.length} project(s), ${comparable} model row(s) `
              + 'carried by 2 or more projects');
  if (projects.length < 2) {
    console.log(`${'no-peer'.padEnd(22)} one project only: this object carries no `
                + 'organization value, so there is nothing to compare against');
    return 0;
  }

  const rows = openaiOutliers(matrix, floor);
  const dimension = { rpm: 'max_requests_per_1_minute', tpm: 'max_tokens_per_1_minute' };
  const seen = new Set();
  for (const [model, pid, dim, value, peerMax] of rows) {
    console.log(`${'project-outlier'.padEnd(22)} ${pid} ${names[pid] ?? '(unnamed)'}  ${model}`);
    console.log(`  ${dimension[dim].padEnd(26)} ${fmt(value)} against a peer maximum `
                + `of ${fmt(peerMax)} (${Math.round((100 * value) / peerMax)}%)`);
    seen.add(`${pid}|${model}`);
  }
  if (rows.length) {
    for (const line of repairLines('project-outlier')) console.log(`  repair: ${line}`);
  }
  return seen.size;
}

async function main() {
  const anthropicKey = process.env.ANTHROPIC_ADMIN_KEY;
  const openaiKey = process.env.OPENAI_ADMIN_KEY;
  if (!anthropicKey && !openaiKey) {
    console.error('set ANTHROPIC_ADMIN_KEY, OPENAI_ADMIN_KEY, or both. Each must be '
                  + 'an organization scoped read credential; a workspace or project '
                  + 'key cannot reach these paths');
    process.exitCode = 2;
    return;
  }
  const floor = Number(process.env.FLOOR ?? 0.5);
  let findings = 0;
  if (anthropicKey) findings += await auditAnthropic(anthropicKey, floor);
  if (openaiKey) findings += await auditOpenai(openaiKey, floor);
  console.log(`${findings} finding(s)`);
  process.exitCode = findings ? 1 : 0;
}

if (import.meta.url === `file://${process.argv[1]}`) await main();

from rate_limit_below_org_audit import (grade_override, group_label,
                                        inherited_limiters, limits_of, num,
                                        openai_matrix, openai_outliers,
                                        org_index, overrides_of, repair_lines,
                                        verdict)


def org_group(models, **limits):
    return {"type": "rate_limit", "group_type": "model_group",
            "models": list(models),
            "limits": [{"type": t, "value": v} for t, v in limits.items()]}


def ws_group(models, limits):
    return {"type": "workspace_rate_limit", "group_type": "model_group",
            "models": list(models), "limits": limits}


def test_a_workspace_capped_at_a_fraction_of_the_org_is_the_finding():
    # The whole note. Both numbers arrive on the same object, so no second
    # lookup can go wrong and no arithmetic is being trusted to a dashboard.
    entry = ws_group(["claude-opus-5"], [
        {"type": "input_tokens_per_minute", "value": 500_000, "org_limit": 10_000_000},
        {"type": "requests_per_minute", "value": 1_000, "org_limit": 4_000},
    ])
    rows = overrides_of(entry)
    assert [r[0] for r in rows] == ["requests_per_minute", "input_tokens_per_minute"]
    state, detail = grade_override(500_000, 10_000_000)
    assert state == "throttled-below-org"
    assert "500,000 of 10,000,000 (5%)" == detail
    assert verdict([s for s, _ in (grade_override(v, o) for _, v, o in rows)]) \
        == "throttled-below-org"
    assert any("Rate limits tab" in line for line in repair_lines(state))


def test_an_override_equal_to_the_org_value_is_a_pin_not_a_no_op():
    # A ratio check passes this at 1.0 and says nothing. It is the state where
    # a container has quietly opted out of every future tier increase.
    state, detail = grade_override(10_000_000, 10_000_000)
    assert state == "override-pinned-at-org"
    assert "will not follow the next increase" in detail
    assert any("Delete the override" in line for line in repair_lines(state))
    # And above the org value is a third thing again: it simply does not apply.
    above, above_detail = grade_override(20_000_000, 10_000_000)
    assert above == "override-above-org"
    assert "applies anyway" in above_detail


def test_a_null_org_limit_is_unjudgeable_and_never_becomes_zero():
    assert num(None) is None and num("nope") is None and num(True) is None
    state, detail = grade_override(500_000, None)
    assert state == "org-limit-unknown"
    assert "cannot be graded" in detail
    assert any("/v1/organizations/rate_limits" in line for line in repair_lines(state))
    # A zero override is the opposite: a real number, and the worst one.
    assert grade_override(0, 10_000_000)[0] == "throttled-below-org"


def test_limiters_absent_from_an_overridden_group_are_reported_as_inherited():
    org = org_index([{"data": [org_group(["claude-opus-5"],
                                         requests_per_minute=4_000,
                                         input_tokens_per_minute=10_000_000,
                                         output_tokens_per_minute=2_000_000)]}])
    label = group_label(org_group(["claude-opus-5"]))
    entry = ws_group(["claude-opus-5"], [
        {"type": "input_tokens_per_minute", "value": 500_000, "org_limit": 10_000_000},
        {"type": "requests_per_minute", "value": 1_000, "org_limit": 4_000},
    ])
    assert inherited_limiters(entry, org[label]) == [
        ("output_tokens_per_minute", 2_000_000)]
    assert verdict(["override-in-range", "limiter-inherited"]) == "limiter-inherited"


def test_the_two_endpoints_label_the_same_group_identically():
    # The join is by label, so this is the assertion the whole Anthropic side
    # rests on: a workspace entry and an organization entry for the same group
    # must produce the same key even though their type fields differ.
    models = ["claude-opus-4-8", "claude-opus-4-5"]
    assert group_label(org_group(models)) == group_label(ws_group(models, []))
    assert group_label(org_group(models)) == "model_group:claude-opus-4-5 +1"
    assert group_label({"group_type": "batch", "models": None}) == "batch"
    assert group_label(None) == "unknown_group"
    assert limits_of({"limits": [{"type": "x", "value": "not-a-number"}]}) == {}


def test_openai_needs_a_peer_because_the_object_has_no_org_value():
    one = openai_matrix({"proj_a": [
        {"model": "gpt-5.6", "max_requests_per_1_minute": 60,
         "max_tokens_per_1_minute": 150_000}]})
    assert openai_outliers(one) == []
    both = openai_matrix({
        "proj_a": [{"model": "gpt-5.6", "max_requests_per_1_minute": 10_000,
                    "max_tokens_per_1_minute": 2_000_000}],
        "proj_b": [{"model": "gpt-5.6", "max_requests_per_1_minute": 9_000,
                    "max_tokens_per_1_minute": 150_000},
                   {"model": "", "max_tokens_per_1_minute": 1}],
    })
    assert sorted(both["gpt-5.6"]) == ["proj_a", "proj_b"]
    assert "" not in both
    rows = openai_outliers(both)
    assert rows == [("gpt-5.6", "proj_b", "tpm", 150_000, 2_000_000)]
    assert any("proxy for the tier" in line
               for line in repair_lines("project-outlier"))


def test_empty_and_absent_inputs_do_not_raise():
    assert org_index(None) == {} and overrides_of(None) == []
    assert openai_matrix(None) == {} and openai_outliers(None) == []
    assert inherited_limiters(None, None) == []
    assert verdict([]) == "no-override" and verdict(None) == "no-override"
    assert grade_override(None, 10)[0] == "no-override"
    assert repair_lines("no-override") == []

import { test } from 'node:test';
import assert from 'node:assert/strict';
import { gradeOverride, groupLabel, inheritedLimiters, limitsOf, num, openaiMatrix,
         openaiOutliers, orgIndex, overridesOf, repairLines, verdict }
  from './rate-limit-below-org-audit.mjs';

const orgGroup = (models, limits) => ({
  type: 'rate_limit', group_type: 'model_group', models: [...models],
  limits: Object.entries(limits ?? {}).map(([type, value]) => ({ type, value })),
});

const wsGroup = (models, limits) => ({
  type: 'workspace_rate_limit', group_type: 'model_group', models: [...models], limits,
});

test('a workspace capped at a fraction of the org is the finding', () => {
  const entry = wsGroup(['claude-opus-5'], [
    { type: 'input_tokens_per_minute', value: 500000, org_limit: 10000000 },
    { type: 'requests_per_minute', value: 1000, org_limit: 4000 },
  ]);
  const rows = overridesOf(entry);
  assert.deepEqual(rows.map((r) => r[0]),
                   ['requests_per_minute', 'input_tokens_per_minute']);
  const [state, detail] = gradeOverride(500000, 10000000);
  assert.equal(state, 'throttled-below-org');
  assert.equal(detail, '500,000 of 10,000,000 (5%)');
  assert.equal(verdict(rows.map((r) => gradeOverride(r[1], r[2])[0])),
               'throttled-below-org');
  assert.ok(repairLines(state).some((l) => l.includes('Rate limits tab')));
});

test('an override equal to the org value is a pin, not a no-op', () => {
  const [state, detail] = gradeOverride(10000000, 10000000);
  assert.equal(state, 'override-pinned-at-org');
  assert.match(detail, /will not follow the next increase/);
  assert.ok(repairLines(state).some((l) => l.includes('Delete the override')));
  const [above, aboveDetail] = gradeOverride(20000000, 10000000);
  assert.equal(above, 'override-above-org');
  assert.match(aboveDetail, /applies anyway/);
});

test('a null org_limit is unjudgeable and never becomes zero', () => {
  assert.equal(num(null), null);
  assert.equal(num('nope'), null);
  assert.equal(num(true), null);
  const [state, detail] = gradeOverride(500000, null);
  assert.equal(state, 'org-limit-unknown');
  assert.match(detail, /cannot be graded/);
  assert.equal(gradeOverride(0, 10000000)[0], 'throttled-below-org');
});

test('limiters absent from an overridden group are reported as inherited', () => {
  const org = orgIndex([{ data: [orgGroup(['claude-opus-5'], {
    requests_per_minute: 4000,
    input_tokens_per_minute: 10000000,
    output_tokens_per_minute: 2000000,
  })] }]);
  const label = groupLabel(orgGroup(['claude-opus-5'], {}));
  const entry = wsGroup(['claude-opus-5'], [
    { type: 'input_tokens_per_minute', value: 500000, org_limit: 10000000 },
    { type: 'requests_per_minute', value: 1000, org_limit: 4000 },
  ]);
  assert.deepEqual(inheritedLimiters(entry, org[label]),
                   [['output_tokens_per_minute', 2000000]]);
  assert.equal(verdict(['override-in-range', 'limiter-inherited']), 'limiter-inherited');
});

test('the two endpoints label the same group identically', () => {
  const models = ['claude-opus-4-8', 'claude-opus-4-5'];
  assert.equal(groupLabel(orgGroup(models, {})), groupLabel(wsGroup(models, [])));
  assert.equal(groupLabel(orgGroup(models, {})), 'model_group:claude-opus-4-5 +1');
  assert.equal(groupLabel({ group_type: 'batch', models: null }), 'batch');
  assert.equal(groupLabel(null), 'unknown_group');
  assert.deepEqual(limitsOf({ limits: [{ type: 'x', value: 'not-a-number' }] }), {});
});

test('openai needs a peer because the object has no org value', () => {
  const one = openaiMatrix({ proj_a: [{ model: 'gpt-5.6',
    max_requests_per_1_minute: 60, max_tokens_per_1_minute: 150000 }] });
  assert.deepEqual(openaiOutliers(one), []);
  const both = openaiMatrix({
    proj_a: [{ model: 'gpt-5.6', max_requests_per_1_minute: 10000,
               max_tokens_per_1_minute: 2000000 }],
    proj_b: [{ model: 'gpt-5.6', max_requests_per_1_minute: 9000,
               max_tokens_per_1_minute: 150000 },
             { model: '', max_tokens_per_1_minute: 1 }],
  });
  assert.deepEqual(Object.keys(both['gpt-5.6'] ?? {}).sort(), ['proj_a', 'proj_b']);
  assert.equal(both[''], undefined);
  assert.deepEqual(openaiOutliers(both),
                   [['gpt-5.6', 'proj_b', 'tpm', 150000, 2000000]]);
  assert.ok(repairLines('project-outlier').some((l) => l.includes('proxy for the tier')));
});

test('empty and absent inputs do not raise', () => {
  assert.deepEqual(orgIndex(null), {});
  assert.deepEqual(overridesOf(null), []);
  assert.deepEqual(openaiMatrix(null), {});
  assert.deepEqual(openaiOutliers(null), []);
  assert.deepEqual(inheritedLimiters(null, null), []);
  assert.equal(verdict([]), 'no-override');
  assert.equal(verdict(null), 'no-override');
  assert.equal(gradeOverride(null, 10)[0], 'no-override');
  assert.deepEqual(repairLines('no-override'), []);
});