bulk issue or comment creation exceeds 80 requests a minute

"""Find bursts of created issues and comments that will trip the content limit.

Read only. Every request is a GET, and the repair is printed rather than run.

Content-generating requests are capped at about 80 a minute and 500 an hour,
separately from the hourly quota, and no API reports how much of that allowance
is left. So this looks at the evidence a bulk writer leaves behind: the density
of created_at timestamps for a single account.
"""
import argparse
import logging
import os
import sys
from datetime import datetime, timezone

import requests

logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s")
log = logging.getLogger("github_content_burst_audit")

API = "https://api.github.com"
UA = "github-content-burst-audit/1.0"

# Documented content-creation ceilings. Both are approximate on GitHub's side and
# neither is exposed as a bucket, which is why they are constants here.
MINUTE_LIMIT = 80
HOUR_LIMIT = 500

# A burst whose newest item is inside this many seconds of now is still running,
# which changes the advice from "pace it before next time" to "stop it".
LIVE_SECONDS = 900


def parse_ts(value):
    """ISO 8601 to epoch seconds, or None. Pure.

    GitHub always sends UTC with a trailing Z. A value that parses without a
    timezone is still treated as UTC rather than as local time, because reading
    the same log on two machines must not produce two different answers.
    """
    text = str(value or "").strip()
    if not text:
        return None
    if text.endswith("Z"):
        text = text[:-1] + "+00:00"
    try:
        moment = datetime.fromisoformat(text)
    except ValueError:
        return None
    if moment.tzinfo is None:
        moment = moment.replace(tzinfo=timezone.utc)
    return moment.timestamp()


def peak_rate(times, window):
    """Most timestamps falling inside any window of that many seconds. Pure.

    Two pointers over a sorted list. Returns (count, ending_at) so the caller can
    say when the densest stretch was, which is what turns a number into something
    someone can go and look at.
    """
    values = sorted(t for t in (times or []) if t is not None)
    peak, at, start = 0, None, 0
    for end in range(len(values)):
        while values[end] - values[start] >= window:
            start += 1
        count = end - start + 1
        if count > peak:
            peak, at = count, values[end]
    return (peak, at)


def by_actor(items):
    """Group created_at timestamps by the login that created them. Pure.

    The limit is per account, so a repository-wide count is the wrong number:
    thirty issues in a minute from thirty people is a triage session, and thirty
    from one login is a script that is about to be throttled.
    """
    out = {}
    for item in items or []:
        user = item.get("user") or {}
        login = str(user.get("login") or "unknown")
        when = parse_ts(item.get("created_at"))
        if when is None:
            continue
        bucket = out.setdefault(login, {"times": [], "type": user.get("type") or "User"})
        bucket["times"].append(when)
    return out


def verdict(peak_minute, peak_hour, last_seen, now):
    """Classify one account's creation pattern. Pure. Returns (state, detail).

    now is a parameter rather than a call to time.time() so the same input always
    produces the same output, and so the tests can put a burst four minutes in the
    past without sleeping for four minutes.
    """
    if not peak_minute:
        return ("quiet", "nothing created in the window that was read")

    age = None if last_seen is None else max(0.0, float(now) - float(last_seen))
    when = ("still running" if age is not None and age < LIVE_SECONDS
            else "already finished" if age is not None
            else "at an unknown time")
    tail = ", %s (newest item %d minute(s) ago)" % (when, int((age or 0) // 60))

    if peak_minute >= MINUTE_LIMIT:
        return ("over-minute",
                "%d created inside one minute against a ceiling of %d. This "
                "account has already been throttled or is about to be%s"
                % (peak_minute, MINUTE_LIMIT, tail))
    if peak_hour >= HOUR_LIMIT:
        return ("over-hour",
                "%d created inside one hour against a ceiling of %d. Pacing "
                "under the per-minute limit is not enough on its own%s"
                % (peak_hour, HOUR_LIMIT, tail))
    if peak_minute >= MINUTE_LIMIT * 0.8:
        return ("near-minute",
                "%d in a minute, %d%% of the ceiling. One issue billed as two "
                "requests puts this over%s"
                % (peak_minute, int(100 * peak_minute / MINUTE_LIMIT), tail))
    if peak_hour >= HOUR_LIMIT * 0.8:
        return ("near-hour",
                "%d in an hour, %d%% of the ceiling. The per-minute rate is fine "
                "and the sustained rate is not%s"
                % (peak_hour, int(100 * peak_hour / HOUR_LIMIT), tail))
    return ("clear",
            "densest minute %d, densest hour %d, both well under %d and %d"
            % (peak_minute, peak_hour, MINUTE_LIMIT, HOUR_LIMIT))


def next_link(response):
    """The rel=next URL from the Link header, or None."""
    for part in (response.headers.get("Link") or "").split(","):
        chunk = part.strip()
        if chunk.startswith("<") and chunk.endswith('rel="next"'):
            return chunk[1:chunk.index(">")]
    return None


def get(session, url, **params):
    r = session.get(url, params=params, timeout=30)
    if r.status_code == 401:
        raise SystemExit("401 from GitHub: GITHUB_TOKEN is missing, expired or "
                         "malformed")
    if r.status_code in (403, 404):
        raise SystemExit("%d from %s: this needs read access to the repository's "
                         "issues. GitHub answers 404 rather than 403 when a "
                         "token cannot see a resource at all."
                         % (r.status_code, url))
    r.raise_for_status()
    return r


def page(session, url, limit, **params):
    out = []
    while url and len(out) < limit:
        r = get(session, url, **params)
        out.extend(r.json())
        url, params = next_link(r), {}
    return out[:limit]


def main():
    ap = argparse.ArgumentParser(description=__doc__)
    ap.add_argument("--repo", required=True, help="owner/name")
    ap.add_argument("--max-items", type=int, default=600,
                    help="stop paging each list after this many items")
    ap.add_argument("--actor", default=None,
                    help="only report this login (default: every author found)")
    args = ap.parse_args()

    token = os.environ.get("GITHUB_TOKEN")
    if not token:
        log.error("set GITHUB_TOKEN (a read-only token is enough)")
        return 2

    owner, _, name = args.repo.partition("/")
    if not (owner and name):
        log.error("--repo takes owner/name, for example acme/api")
        return 2

    session = requests.Session()
    session.headers.update({
        "Authorization": "Bearer " + token,
        "Accept": "application/vnd.github+json",
        "X-GitHub-Api-Version": "2022-11-28",
        "User-Agent": UA,
    })

    base = "%s/repos/%s/%s" % (API, owner, name)
    items = page(session, base + "/issues", args.max_items, state="all",
                 sort="created", direction="desc", per_page=100)
    items += page(session, base + "/issues/comments", args.max_items,
                  sort="created", direction="desc", per_page=100)
    log.info("read %d issue(s), pull request(s) and comment(s) on %s",
             len(items), args.repo)

    now = datetime.now(timezone.utc).timestamp()
    findings = 0
    actors = by_actor(items)
    for login, bucket in sorted(actors.items(),
                                key=lambda kv: -len(kv[1]["times"])):
        if args.actor and login != args.actor:
            continue
        times = bucket["times"]
        peak_minute, minute_at = peak_rate(times, 60)
        peak_hour, _ = peak_rate(times, 3600)
        state, detail = verdict(peak_minute, peak_hour,
                                max(times) if times else None, now)
        line = "%s (%s): %s" % (login, bucket["type"], detail)
        if state in ("clear", "quiet"):
            log.info(line)
            continue
        findings += 1
        log.warning(line)
        if minute_at:
            log.warning("  densest minute ended at %s",
                        datetime.fromtimestamp(minute_at, timezone.utc).isoformat())
        log.warning("  repair: pace this writer to one creating request per "
                    "second and under 300 an hour, sleeping between items "
                    "rather than relying on the network being slow.")
        log.warning("  repair: on a 403 carrying retry-after, pause every "
                    "worker for that many seconds instead of retrying the one "
                    "item, and checkpoint what was created so a resume does "
                    "not duplicate it.")

    log.info("%d author(s) examined, %d over or near a content-creation ceiling",
             len(actors) if not args.actor else 1, findings)
    return 1 if findings else 0


if __name__ == "__main__":
    sys.exit(main())

/**
 * Find bursts of created issues and comments that will trip the content limit.
 *
 * Read only. Every request is a GET, and the repair is printed rather than run.
 *
 * Content-generating requests are capped at about 80 a minute and 500 an hour,
 * separately from the hourly quota, and no API reports the remaining allowance.
 */
const API = 'https://api.github.com';
const UA = 'github-content-burst-audit/1.0';

export const MINUTE_LIMIT = 80;
export const HOUR_LIMIT = 500;

// A burst whose newest item is inside this many seconds of now is still running.
const LIVE_SECONDS = 900;

/** ISO 8601 to epoch seconds, or null. Pure. */
export function parseTs(value) {
  const text = String(value ?? '').trim();
  if (!text) return null;
  const ms = Date.parse(text);
  return Number.isFinite(ms) ? ms / 1000 : null;
}

/**
 * Most timestamps falling inside any window of that many seconds. Pure.
 * Two pointers over a sorted list. Returns [count, endingAt].
 */
export function peakRate(times, window) {
  const values = (times ?? []).filter((t) => t !== null && t !== undefined)
    .map(Number).sort((a, b) => a - b);
  let peak = 0;
  let at = null;
  let start = 0;
  for (let end = 0; end < values.length; end += 1) {
    while (values[end] - values[start] >= window) start += 1;
    const count = end - start + 1;
    if (count > peak) { peak = count; at = values[end]; }
  }
  return [peak, at];
}

/**
 * Group created_at timestamps by the login that created them. Pure.
 * The limit is per account, so a repository-wide count is the wrong number.
 */
export function byActor(items) {
  const out = {};
  for (const item of items ?? []) {
    const user = item.user ?? {};
    const login = String(user.login ?? 'unknown');
    const when = parseTs(item.created_at);
    if (when === null) continue;
    const bucket = (out[login] ??= { times: [], type: user.type ?? 'User' });
    bucket.times.push(when);
  }
  return out;
}

/**
 * Classify one account's creation pattern. Pure. Returns [state, detail].
 * now is a parameter so the same input always produces the same output.
 */
export function verdict(peakMinute, peakHour, lastSeen, now) {
  if (!peakMinute) return ['quiet', 'nothing created in the window that was read'];

  const age = lastSeen === null || lastSeen === undefined
    ? null : Math.max(0, Number(now) - Number(lastSeen));
  const when = age === null ? 'at an unknown time'
    : age < LIVE_SECONDS ? 'still running' : 'already finished';
  const tail = `, ${when} (newest item ${Math.floor((age ?? 0) / 60)} minute(s) ago)`;

  if (peakMinute >= MINUTE_LIMIT) {
    return ['over-minute',
      `${peakMinute} created inside one minute against a ceiling of ${MINUTE_LIMIT}. ` +
      `This account has already been throttled or is about to be${tail}`];
  }
  if (peakHour >= HOUR_LIMIT) {
    return ['over-hour',
      `${peakHour} created inside one hour against a ceiling of ${HOUR_LIMIT}. ` +
      `Pacing under the per-minute limit is not enough on its own${tail}`];
  }
  if (peakMinute >= MINUTE_LIMIT * 0.8) {
    return ['near-minute',
      `${peakMinute} in a minute, ${Math.floor(100 * peakMinute / MINUTE_LIMIT)}% of ` +
      `the ceiling. One issue billed as two requests puts this over${tail}`];
  }
  if (peakHour >= HOUR_LIMIT * 0.8) {
    return ['near-hour',
      `${peakHour} in an hour, ${Math.floor(100 * peakHour / HOUR_LIMIT)}% of the ` +
      `ceiling. The per-minute rate is fine and the sustained rate is not${tail}`];
  }
  return ['clear',
    `densest minute ${peakMinute}, densest hour ${peakHour}, both well under ` +
    `${MINUTE_LIMIT} and ${HOUR_LIMIT}`];
}

function nextLink(res) {
  for (const part of (res.headers.get('link') ?? '').split(',')) {
    const chunk = part.trim();
    if (chunk.startsWith('<') && chunk.endsWith('rel="next"')) {
      return chunk.slice(1, chunk.indexOf('>'));
    }
  }
  return null;
}

async function get(token, url) {
  const res = await fetch(url, {
    headers: {
      Authorization: `Bearer ${token}`,
      Accept: 'application/vnd.github+json',
      'X-GitHub-Api-Version': '2022-11-28',
      'User-Agent': UA,
    },
  });
  if (res.status === 401) {
    throw new Error('401 from GitHub: GITHUB_TOKEN is missing, expired or malformed');
  }
  if (res.status === 403 || res.status === 404) {
    throw new Error(`${res.status} from ${url}: this needs read access to the ` +
      "repository's issues. GitHub answers 404 rather than 403 when a token " +
      'cannot see a resource at all.');
  }
  if (!res.ok) throw new Error(`${res.status} from ${url}`);
  return res;
}

async function page(token, url, limit) {
  const out = [];
  let next = url;
  while (next && out.length < limit) {
    const res = await get(token, next);
    out.push(...(await res.json()));
    next = nextLink(res);
  }
  return out.slice(0, limit);
}

async function main() {
  const repo = process.argv[2];
  const token = process.env.GITHUB_TOKEN;
  if (!token) {
    console.error('set GITHUB_TOKEN (a read-only token is enough)');
    process.exitCode = 2;
    return;
  }
  if (!repo || !repo.includes('/')) {
    console.error('usage: node github-content-burst-audit.mjs owner/name');
    process.exitCode = 2;
    return;
  }

  const base = `${API}/repos/${repo}`;
  const limit = 600;
  const items = [
    ...await page(token,
      `${base}/issues?state=all&sort=created&direction=desc&per_page=100`, limit),
    ...await page(token,
      `${base}/issues/comments?sort=created&direction=desc&per_page=100`, limit),
  ];
  console.log(`read ${items.length} issue(s), pull request(s) and comment(s) on ${repo}`);

  const now = Date.now() / 1000;
  const actors = byActor(items);
  let findings = 0;
  const ranked = Object.entries(actors).sort((a, b) => b[1].times.length - a[1].times.length);
  for (const [login, bucket] of ranked) {
    const [peakMinute, minuteAt] = peakRate(bucket.times, 60);
    const [peakHour] = peakRate(bucket.times, 3600);
    const lastSeen = bucket.times.length ? Math.max(...bucket.times) : null;
    const [state, detail] = verdict(peakMinute, peakHour, lastSeen, now);
    const line = `${login} (${bucket.type}): ${detail}`;
    if (state === 'clear' || state === 'quiet') { console.log(line); continue; }
    findings += 1;
    console.warn(line);
    if (minuteAt) {
      console.warn(`  densest minute ended at ${new Date(minuteAt * 1000).toISOString()}`);
    }
    console.warn('  repair: pace this writer to one creating request per second ' +
      'and under 300 an hour, sleeping between items rather than relying on the ' +
      'network being slow.');
    console.warn('  repair: on a 403 carrying retry-after, pause every worker ' +
      'for that many seconds instead of retrying the one item, and checkpoint ' +
      'what was created so a resume does not duplicate it.');
  }

  console.log(`${ranked.length} author(s) examined, ${findings} over or near a ` +
    'content-creation ceiling');
  process.exitCode = findings ? 1 : 0;
}

// Only run when invoked directly. The test file imports this module, and without
// the guard main() would run there too, fail on the missing token, and set a
// non-zero exit code that fails the whole test file even as every test passes.
if (import.meta.url === `file://${process.argv[1]}`) {
  main().catch((err) => { console.error(err.message); process.exitCode = 2; });
}

from github_content_burst_audit import by_actor, parse_ts, peak_rate, verdict

NOW = 1756512000.0  # 2025-08-30T00:00:00Z, so every case below is anchored.


def issue(login, created_at, kind="Bot"):
    return {"user": {"login": login, "type": kind}, "created_at": created_at}


def test_parse_ts_reads_githubs_z_suffix():
    assert parse_ts("2025-08-30T00:00:00Z") == NOW


def test_parse_ts_returns_none_rather_than_raising():
    assert parse_ts(None) is None
    assert parse_ts("") is None
    assert parse_ts("last tuesday") is None


def test_a_naive_timestamp_is_read_as_utc_not_as_local_time():
    # Two machines in two timezones must not disagree about the same log.
    assert parse_ts("2025-08-30T00:00:00") == NOW


def test_peak_rate_of_a_steady_trickle_is_one_per_window():
    times = [NOW + 120 * i for i in range(10)]
    peak, _ = peak_rate(times, 60)
    assert peak == 1


def test_peak_rate_finds_the_burst_and_says_when_it_ended():
    times = [NOW + i for i in range(90)] + [NOW + 10000]
    peak, at = peak_rate(times, 60)
    assert peak == 60
    assert at == NOW + 59


def test_the_window_edge_is_exclusive_so_a_full_minute_counts_once():
    assert peak_rate([NOW, NOW + 60], 60)[0] == 1
    assert peak_rate([NOW, NOW + 59.9], 60)[0] == 2


def test_peak_rate_of_nothing_is_zero():
    assert peak_rate([], 60) == (0, None)
    assert peak_rate(None, 60) == (0, None)


def test_by_actor_groups_per_login_and_keeps_the_account_type():
    grouped = by_actor([issue("bot", "2025-08-30T00:00:00Z"),
                        issue("bot", "2025-08-30T00:00:01Z"),
                        issue("person", "2025-08-30T00:00:02Z", "User")])
    assert sorted(grouped) == ["bot", "person"]
    assert len(grouped["bot"]["times"]) == 2
    assert grouped["person"]["type"] == "User"


def test_by_actor_drops_items_with_no_readable_timestamp():
    grouped = by_actor([issue("bot", None), issue("bot", "2025-08-30T00:00:00Z")])
    assert len(grouped["bot"]["times"]) == 1


def test_eighty_in_a_minute_is_the_finding():
    state, detail = verdict(80, 80, NOW, NOW)
    assert state == "over-minute"
    assert "still running" in detail


def test_a_burst_that_finished_hours_ago_is_reported_as_finished():
    state, detail = verdict(90, 90, NOW - 7200, NOW)
    assert state == "over-minute"
    assert "already finished" in detail
    assert "120 minute(s) ago" in detail


def test_a_gentle_rate_can_still_break_the_hourly_ceiling():
    # Ten a minute never trips the per-minute limit and is 600 an hour.
    state, detail = verdict(10, 600, NOW, NOW)
    assert state == "over-hour"
    assert "per-minute limit is not enough" in detail


def test_the_near_states_warn_before_the_ceiling():
    assert verdict(64, 64, NOW, NOW)[0] == "near-minute"
    assert verdict(10, 400, NOW, NOW)[0] == "near-hour"


def test_an_ordinary_repository_is_clear():
    state, _ = verdict(3, 40, NOW, NOW)
    assert state == "clear"


def test_no_activity_is_quiet_rather_than_clear():
    assert verdict(0, 0, None, NOW)[0] == "quiet"

import { test } from 'node:test';
import assert from 'node:assert/strict';
import {
  byActor, parseTs, peakRate, verdict,
} from './github-content-burst-audit.mjs';

const NOW = 1756512000; // 2025-08-30T00:00:00Z, so every case below is anchored.

const issue = (login, created_at, type = 'Bot') => ({ user: { login, type }, created_at });

test('parseTs reads the Z suffix GitHub sends', () => {
  assert.equal(parseTs('2025-08-30T00:00:00Z'), NOW);
});

test('parseTs returns null rather than throwing', () => {
  assert.equal(parseTs(null), null);
  assert.equal(parseTs(''), null);
  assert.equal(parseTs('last tuesday'), null);
});

test('peakRate of a steady trickle is one per window', () => {
  const times = Array.from({ length: 10 }, (_, i) => NOW + 120 * i);
  assert.equal(peakRate(times, 60)[0], 1);
});

test('peakRate finds the burst and says when it ended', () => {
  const times = [...Array.from({ length: 90 }, (_, i) => NOW + i), NOW + 10000];
  const [peak, at] = peakRate(times, 60);
  assert.equal(peak, 60);
  assert.equal(at, NOW + 59);
});

test('the window edge is exclusive so a full minute counts once', () => {
  assert.equal(peakRate([NOW, NOW + 60], 60)[0], 1);
  assert.equal(peakRate([NOW, NOW + 59.9], 60)[0], 2);
});

test('peakRate of nothing is zero', () => {
  assert.deepEqual(peakRate([], 60), [0, null]);
  assert.deepEqual(peakRate(null, 60), [0, null]);
});

test('byActor groups per login and keeps the account type', () => {
  const grouped = byActor([
    issue('bot', '2025-08-30T00:00:00Z'),
    issue('bot', '2025-08-30T00:00:01Z'),
    issue('person', '2025-08-30T00:00:02Z', 'User'),
  ]);
  assert.deepEqual(Object.keys(grouped).sort(), ['bot', 'person']);
  assert.equal(grouped.bot.times.length, 2);
  assert.equal(grouped.person.type, 'User');
});

test('byActor drops items with no readable timestamp', () => {
  const grouped = byActor([issue('bot', null), issue('bot', '2025-08-30T00:00:00Z')]);
  assert.equal(grouped.bot.times.length, 1);
});

test('eighty in a minute is the finding', () => {
  const [state, detail] = verdict(80, 80, NOW, NOW);
  assert.equal(state, 'over-minute');
  assert.match(detail, /still running/);
});

test('a burst that finished hours ago is reported as finished', () => {
  const [state, detail] = verdict(90, 90, NOW - 7200, NOW);
  assert.equal(state, 'over-minute');
  assert.match(detail, /already finished/);
  assert.match(detail, /120 minute/);
});

test('a gentle rate can still break the hourly ceiling', () => {
  const [state, detail] = verdict(10, 600, NOW, NOW);
  assert.equal(state, 'over-hour');
  assert.match(detail, /per-minute limit is not enough/);
});

test('the near states warn before the ceiling', () => {
  assert.equal(verdict(64, 64, NOW, NOW)[0], 'near-minute');
  assert.equal(verdict(10, 400, NOW, NOW)[0], 'near-hour');
});

test('an ordinary repository is clear', () => {
  assert.equal(verdict(3, 40, NOW, NOW)[0], 'clear');
});

test('no activity is quiet rather than clear', () => {
  assert.equal(verdict(0, 0, null, NOW)[0], 'quiet');
});