per_page is unset so every list costs 3.3x more requests

"""Report how many requests an unset per_page is costing on each list endpoint.

Read only. GET requests and nothing else: a token with read access is enough.
The repair is printed, never performed.

This is a cost check, not a correctness one. Raising per_page does not make a
client that ignores the Link header correct; it makes it wrong by 100 instead
of by 30.
"""
import argparse
import logging
import os
import re
import sys
from urllib.parse import parse_qs, urlparse

import requests

logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s")
log = logging.getLogger("github_per_page_audit")

API = "https://api.github.com"
LINK = re.compile(r'<([^>]+)>\s*;\s*rel="([^"]+)"')

MAX_PER_PAGE = 100
DEFAULT_PER_PAGE = 30

PROBES = [
    ("issues", {"state": "all"}),
    ("pulls", {"state": "all"}),
    ("commits", {}),
    ("branches", {}),
    ("tags", {}),
]


def pages_for(items, per_page):
    """Requests needed to read `items` at `per_page`. Pure.

    Clamps to 100 the way the API does rather than the way the caller hoped:
    per_page above the maximum is silently reduced, not rejected, so pretending
    500 works here would hide exactly the mistake this script exists to find.
    """
    size = min(max(int(per_page or DEFAULT_PER_PAGE), 1), MAX_PER_PAGE)
    items = int(items or 0)
    if items <= 0:
        return 0
    return -(-items // size)


def verdict(items, per_page=DEFAULT_PER_PAGE):
    """Classify one endpoint's page-size arithmetic. Pure. Returns (state, detail)."""
    items = int(items or 0)
    if items <= 0:
        return ("empty", "no items; nothing to page and nothing to save")

    now = pages_for(items, per_page)
    best = pages_for(items, MAX_PER_PAGE)

    if now == best:
        if int(per_page or DEFAULT_PER_PAGE) > MAX_PER_PAGE:
            return ("at-maximum",
                    "%d item(s) in %d request(s). per_page=%s is above the maximum "
                    "and was clamped to 100, which costs nothing here but will "
                    "mislead any loop that trusts the number it asked for."
                    % (items, now, per_page))
        return ("at-maximum" if now > 1 else "single-page",
                "%d item(s) in %d request(s); per_page=100 would not improve on it."
                % (items, now))

    saved = now - best
    return ("wasteful",
            "%d item(s): %d request(s) at per_page=%d, %d at per_page=100. "
            "%d request(s) of quota and %d round trip(s) wasted on every full "
            "pass (%.0f%%)."
            % (items, now, int(per_page), best, saved, saved, 100.0 * saved / now))


def parse_link(header):
    if not header:
        return {}
    return {rel: url for url, rel in LINK.findall(header)}


def page_number(url):
    if not url:
        return None
    values = parse_qs(urlparse(url).query).get("page") or []
    try:
        return int(values[0])
    except (IndexError, TypeError, ValueError):
        return None


def get(session, path, **params):
    r = session.get(API + path, params=params, timeout=30)
    if r.status_code == 401:
        raise SystemExit("401 from GitHub: GITHUB_TOKEN is missing, malformed or "
                         "revoked")
    if r.status_code == 403 and "rate limit" in r.text.lower():
        raise SystemExit("403 rate limited. GET /rate_limit reports the reset time "
                         "and does not itself consume quota")
    r.raise_for_status()
    return r


def count_items(session, path, extra):
    """Exact item count in at most two requests.

    Page one at the maximum page size gives rel="last"; reading that last page
    gives the remainder. (last - 1) * 100 + len(last page) is the total, with no
    estimation anywhere in it.
    """
    first = get(session, path, per_page=MAX_PER_PAGE, **extra)
    body = first.json()
    if not isinstance(body, list):
        return None
    last = page_number(parse_link(first.headers.get("Link")).get("last"))
    if last is None or last <= 1:
        return len(body)
    tail = get(session, path, per_page=MAX_PER_PAGE, page=last, **extra).json()
    return (last - 1) * MAX_PER_PAGE + len(tail)


def main():
    ap = argparse.ArgumentParser(description=__doc__)
    ap.add_argument("--repo", required=True, help="owner/name")
    ap.add_argument("--per-page", type=int, default=DEFAULT_PER_PAGE,
                    help="the page size your client currently sends "
                         "(default 30, which is what an unset per_page means)")
    args = ap.parse_args()

    token = os.environ.get("GITHUB_TOKEN")
    if not token:
        log.error("set GITHUB_TOKEN (a read-only token is enough)")
        return 2

    session = requests.Session()
    session.headers.update({
        "Authorization": "Bearer " + token,
        "Accept": "application/vnd.github+json",
        "X-GitHub-Api-Version": "2022-11-28",
        "User-Agent": "github-per-page-audit",
    })

    wasteful = 0
    recoverable = 0
    for name, extra in PROBES:
        path = "/repos/%s/%s" % (args.repo, name)
        items = count_items(session, path, extra)
        if items is None:
            log.info("%-12s %s  not a list endpoint, skipped", "skipped", path)
            continue
        state, detail = verdict(items, args.per_page)
        line = "%-12s %s  %s" % (state, path, detail)
        if state == "wasteful":
            wasteful += 1
            recoverable += pages_for(items, args.per_page) - pages_for(items, MAX_PER_PAGE)
            log.warning(line)
            log.warning("  repair: add per_page=100 to this request. It returns the "
                        "same data for the same one request per page.")
        else:
            log.info(line)

    log.info("%d endpoint(s), %d wasteful, %d request(s) per pass recoverable",
             len(PROBES), wasteful, recoverable)
    return 1 if wasteful else 0


if __name__ == "__main__":
    sys.exit(main())

/**
 * Report how many requests an unset per_page is costing on each list endpoint.
 *
 * Read only. GET requests and nothing else: a token with read access is enough.
 * The repair is printed, never performed.
 *
 * A cost check, not a correctness one. Raising per_page does not make a client
 * that ignores the Link header correct.
 */
const API = 'https://api.github.com';
const LINK = /<([^>]+)>\s*;\s*rel="([^"]+)"/g;

const MAX_PER_PAGE = 100;
const DEFAULT_PER_PAGE = 30;

const PROBES = [
  ['issues', { state: 'all' }],
  ['pulls', { state: 'all' }],
  ['commits', {}],
  ['branches', {}],
  ['tags', {}],
];

/**
 * Requests needed to read `items` at `perPage`. Pure. Clamps at 100 the way the
 * API does, because per_page above the maximum is reduced rather than rejected.
 */
export function pagesFor(items, perPage) {
  const size = Math.min(Math.max(Number(perPage) || DEFAULT_PER_PAGE, 1), MAX_PER_PAGE);
  const n = Number(items) || 0;
  return n <= 0 ? 0 : Math.ceil(n / size);
}

/** Classify one endpoint's page-size arithmetic. Pure. Returns [state, detail]. */
export function verdict(items, perPage = DEFAULT_PER_PAGE) {
  const n = Number(items) || 0;
  if (n <= 0) return ['empty', 'no items; nothing to page and nothing to save'];

  const now = pagesFor(n, perPage);
  const best = pagesFor(n, MAX_PER_PAGE);

  if (now === best) {
    if ((Number(perPage) || DEFAULT_PER_PAGE) > MAX_PER_PAGE) {
      return ['at-maximum',
        `${n} item(s) in ${now} request(s). per_page=${perPage} is above the ` +
        'maximum and was clamped to 100, which costs nothing here but will ' +
        'mislead any loop that trusts the number it asked for.'];
    }
    return [now > 1 ? 'at-maximum' : 'single-page',
      `${n} item(s) in ${now} request(s); per_page=100 would not improve on it.`];
  }

  const saved = now - best;
  const pct = Math.round((100 * saved) / now);
  return ['wasteful',
    `${n} item(s): ${now} request(s) at per_page=${perPage}, ${best} at ` +
    `per_page=100. ${saved} request(s) of quota and ${saved} round trip(s) ` +
    `wasted on every full pass (${pct}%).`];
}

function parseLink(header) {
  const out = new Map();
  if (!header) return out;
  for (const m of String(header).matchAll(LINK)) out.set(m[2], m[1]);
  return out;
}

function pageNumber(url) {
  if (!url) return null;
  const value = new URL(url, API).searchParams.get('page');
  const n = Number(value);
  return value !== null && Number.isInteger(n) ? n : null;
}

function arg(name, fallback) {
  const i = process.argv.indexOf(`--${name}`);
  return i === -1 ? fallback : process.argv[i + 1];
}

async function get(token, path, params = {}) {
  const url = new URL(API + path);
  for (const [k, v] of Object.entries(params)) url.searchParams.set(k, v);
  const res = await fetch(url, {
    headers: {
      Authorization: `Bearer ${token}`,
      Accept: 'application/vnd.github+json',
      'X-GitHub-Api-Version': '2022-11-28',
      'User-Agent': 'github-per-page-audit',
    },
  });
  if (res.status === 401) {
    throw new Error('401 from GitHub: GITHUB_TOKEN is missing, malformed or revoked');
  }
  if (!res.ok) throw new Error(`${res.status} from ${url.pathname}`);
  return res;
}

async function countItems(token, path, extra) {
  const first = await get(token, path, { per_page: MAX_PER_PAGE, ...extra });
  const body = await first.json();
  if (!Array.isArray(body)) return null;
  const last = pageNumber(parseLink(first.headers.get('link')).get('last'));
  if (last === null || last <= 1) return body.length;
  const tail = await (await get(token, path,
    { per_page: MAX_PER_PAGE, page: last, ...extra })).json();
  return (last - 1) * MAX_PER_PAGE + tail.length;
}

async function main() {
  const token = process.env.GITHUB_TOKEN;
  const repo = arg('repo');
  if (!token || !repo) {
    console.error('set GITHUB_TOKEN and pass --repo owner/name');
    process.exitCode = 2;
    return;
  }
  const perPage = Number(arg('per-page', DEFAULT_PER_PAGE)) || DEFAULT_PER_PAGE;

  let wasteful = 0;
  let recoverable = 0;
  for (const [name, extra] of PROBES) {
    const path = `/repos/${repo}/${name}`;
    const items = await countItems(token, path, extra);
    if (items === null) {
      console.log(`skipped      ${path}  not a list endpoint, skipped`);
      continue;
    }
    const [state, detail] = verdict(items, perPage);
    const line = `${state.padEnd(12)} ${path}  ${detail}`;
    if (state === 'wasteful') {
      wasteful += 1;
      recoverable += pagesFor(items, perPage) - pagesFor(items, MAX_PER_PAGE);
      console.warn(line);
      console.warn('  repair: add per_page=100 to this request. It returns the ' +
                   'same data for the same one request per page.');
    } else {
      console.log(line);
    }
  }

  console.log(`${PROBES.length} endpoint(s), ${wasteful} wasteful, ` +
              `${recoverable} request(s) per pass recoverable`);
  process.exitCode = wasteful ? 1 : 0;
}

// Only run when invoked directly, so importing this module from the test file
// does not run main(), fail on the missing token and fail the whole suite.
if (import.meta.url === `file://${process.argv[1]}`) {
  main().catch((err) => { console.error(err.message); process.exitCode = 2; });
}

from github_per_page_audit import pages_for, verdict


def test_page_count_is_a_ceiling_not_a_division():
    assert pages_for(3000, 30) == 100
    assert pages_for(3000, 100) == 30
    assert pages_for(3001, 100) == 31
    assert pages_for(1, 100) == 1


def test_per_page_above_the_maximum_is_clamped_to_100():
    # The API reduces it silently rather than rejecting it, so the arithmetic
    # has to reduce it too or the saving reported here is fiction.
    assert pages_for(3000, 500) == 30
    state, detail = verdict(3000, 500)
    assert state == "at-maximum"
    assert "clamped" in detail


def test_the_default_page_size_is_the_finding():
    state, detail = verdict(3412, 30)
    assert state == "wasteful"
    assert "114 request(s) at per_page=30" in detail
    assert "35 at per_page=100" in detail
    assert "79 request(s)" in detail


def test_a_full_page_size_has_nothing_to_recover():
    assert verdict(3412, 100)[0] == "at-maximum"


def test_a_short_list_is_one_request_either_way():
    state, _ = verdict(12, 30)
    assert state == "single-page"


def test_an_empty_collection_is_not_reported_as_wasteful():
    assert verdict(0, 30)[0] == "empty"
    assert pages_for(0, 30) == 0

import { test } from 'node:test';
import assert from 'node:assert/strict';
import { pagesFor, verdict } from './github-per-page-audit.mjs';

test('page count is a ceiling, not a division', () => {
  assert.equal(pagesFor(3000, 30), 100);
  assert.equal(pagesFor(3000, 100), 30);
  assert.equal(pagesFor(3001, 100), 31);
  assert.equal(pagesFor(1, 100), 1);
});

test('per_page above the maximum is clamped to 100', () => {
  assert.equal(pagesFor(3000, 500), 30);
  const [state, detail] = verdict(3000, 500);
  assert.equal(state, 'at-maximum');
  assert.match(detail, /clamped/);
});

test('the default page size is the finding', () => {
  const [state, detail] = verdict(3412, 30);
  assert.equal(state, 'wasteful');
  assert.match(detail, /114 request\(s\) at per_page=30/);
  assert.match(detail, /35 at per_page=100/);
  assert.match(detail, /79 request\(s\)/);
});

test('a full page size has nothing to recover', () => {
  assert.equal(verdict(3412, 100)[0], 'at-maximum');
});

test('a short list is one request either way', () => {
  assert.equal(verdict(12, 30)[0], 'single-page');
});

test('an empty collection is not reported as wasteful', () => {
  assert.equal(verdict(0, 30)[0], 'empty');
  assert.equal(pagesFor(0, 30), 0);
});