only the first page is read because the Link header is ignored

"""Report GitHub list endpoints that advertise pages your client may not read.

Read only. GET requests and nothing else: a token with read access to the
repository is enough, and that is what you should give it. The repair is printed,
never performed.

What this can and cannot see: the API has no idea whether your client follows
rel="next". It can only say whether there is a next page there to be missed, and
how many items are on the far side of it. That is the trap, not the fall.
"""
import argparse
import logging
import os
import re
import sys
from urllib.parse import parse_qs, urlparse

import requests

logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s")
log = logging.getLogger("github_link_header_audit")

API = "https://api.github.com"

# Anchored on the angle brackets rather than split on ",". A pagination URL can
# contain a comma of its own -- labels=bug,ci is the everyday case -- and
# splitting the header on commas turns one good link into two broken ones.
LINK = re.compile(r'<([^>]+)>\s*;\s*rel="([^"]+)"')

PROBES = [
    ("pulls", {"state": "open"}),
    ("issues", {"state": "all"}),
    ("branches", {}),
    ("tags", {}),
    ("contributors", {}),
]


def parse_link(header):
    """Parse a Link header into {rel: url}. Pure, so it is tested offline."""
    if not header:
        return {}
    return {rel: url for url, rel in LINK.findall(header)}


def page_number(url):
    """Read the page query parameter out of a pagination URL, or None."""
    if not url:
        return None
    values = parse_qs(urlparse(url).query).get("page") or []
    try:
        return int(values[0])
    except (IndexError, TypeError, ValueError):
        return None


def verdict(links, received, per_page=1):
    """Classify what one list response says about its own completeness.

    Pure, so the rules are visible rather than buried in a request loop.
    Returns (state, detail).

    The states are deliberately three and not two. "more-pages-unsized" is the
    case where rel="next" exists and rel="last" does not: the list is still
    truncated, and a loop that terminates on the missing rel="last" is the same
    bug this note is about.
    """
    if "next" not in links:
        return ("single-page",
                '%d item(s) and no rel="next". One request really is the whole '
                "list here." % received)

    last = page_number(links.get("last"))
    if last is None:
        return ("more-pages-unsized",
                'rel="next" is present and rel="last" is not, so the total is only '
                "knowable by walking it. Terminate on the absence of "
                'rel="next", never on the absence of rel="last".')

    if per_page == 1:
        return ("more-pages",
                "%d item(s) in total. A client that reads the first page and stops "
                "reports %d." % (last, received))

    return ("more-pages",
            "%d page(s) at per_page=%d, so %d to %d item(s) in total. A client "
            "that reads the first page and stops reports %d."
            % (last, per_page, (last - 1) * per_page + 1, last * per_page, received))


def get(session, path, **params):
    r = session.get(API + path, params=params, timeout=30)
    if r.status_code == 401:
        raise SystemExit("401 from GitHub: GITHUB_TOKEN is missing, malformed or "
                         "revoked")
    if r.status_code == 403 and "rate limit" in r.text.lower():
        raise SystemExit("403 rate limited. GET /rate_limit reports the reset time "
                         "and does not itself consume quota")
    if r.status_code == 404:
        raise SystemExit("404 on %s: the repository does not exist, or this token "
                         "cannot see it -- GitHub returns 404 rather than 403 for "
                         "resources you may not know about" % path)
    r.raise_for_status()
    return r


def main():
    ap = argparse.ArgumentParser(description=__doc__)
    ap.add_argument("--repo", required=True, help="owner/name")
    ap.add_argument("--path", action="append",
                    help="probe this API path instead of the defaults, e.g. "
                         "/repos/o/n/releases. Repeatable.")
    args = ap.parse_args()

    token = os.environ.get("GITHUB_TOKEN")
    if not token:
        log.error("set GITHUB_TOKEN (a read-only token is enough)")
        return 2

    session = requests.Session()
    session.headers.update({
        "Authorization": "Bearer " + token,
        "Accept": "application/vnd.github+json",
        "X-GitHub-Api-Version": "2022-11-28",
        # GitHub rejects requests with no User-Agent outright.
        "User-Agent": "github-link-header-audit",
    })

    if args.path:
        probes = [(p, {}) for p in args.path]
    else:
        probes = [("/repos/%s/%s" % (args.repo, name), extra)
                  for name, extra in PROBES]

    truncatable = 0
    remaining = "?"
    for path, extra in probes:
        # per_page=1 makes the rel="last" page number the exact item count, for
        # one request and one item of transfer.
        r = get(session, path, per_page=1, **extra)
        remaining = r.headers.get("x-ratelimit-remaining", "?")
        body = r.json()
        received = len(body) if isinstance(body, list) else 0
        state, detail = verdict(parse_link(r.headers.get("Link")), received, 1)

        line = "%-18s %s  %s" % (state, path, detail)
        if state == "single-page":
            log.info(line)
            continue
        truncatable += 1
        log.warning(line)
        log.warning('  repair: follow rel="next" until it is absent -- '
                    "octokit.paginate() in Octokit, the PaginatedList in PyGithub, "
                    "gh api --paginate on the command line. Never build page URLs "
                    "by hand.")

    log.info("%d endpoint(s) probed, %d with pages beyond the first; "
             "x-ratelimit-remaining %s", len(probes), truncatable, remaining)
    return 1 if truncatable else 0


if __name__ == "__main__":
    sys.exit(main())

/**
 * Report GitHub list endpoints that advertise pages your client may not read.
 *
 * Read only. GET requests and nothing else: a token with read access to the
 * repository is enough. The repair is printed, never performed.
 *
 * The API cannot see whether your client follows rel="next". It can only say
 * whether there is a next page there to be missed.
 */
const API = 'https://api.github.com';

// Anchored on the angle brackets rather than split on ','. A pagination URL can
// contain a comma of its own (labels=bug,ci) and splitting the header on commas
// turns one good link into two broken ones.
const LINK = /<([^>]+)>\s*;\s*rel="([^"]+)"/g;

const PROBES = [
  ['pulls', { state: 'open' }],
  ['issues', { state: 'all' }],
  ['branches', {}],
  ['tags', {}],
  ['contributors', {}],
];

/** Parse a Link header into a Map of rel to url. Pure, so it is tested offline. */
export function parseLink(header) {
  const out = new Map();
  if (!header) return out;
  for (const m of String(header).matchAll(LINK)) out.set(m[2], m[1]);
  return out;
}

/** Read the page query parameter out of a pagination URL, or null. */
export function pageNumber(url) {
  if (!url) return null;
  let value;
  try {
    value = new URL(url, API).searchParams.get('page');
  } catch {
    return null;
  }
  const n = Number(value);
  return value !== null && Number.isInteger(n) ? n : null;
}

/**
 * Classify what one list response says about its own completeness. Pure.
 * Returns [state, detail].
 *
 * Three states, not two: a rel="next" with no rel="last" is still a truncated
 * list, and a loop that stops there has the same bug in a different costume.
 */
export function verdict(links, received, perPage = 1) {
  if (!links.has('next')) {
    return ['single-page',
      `${received} item(s) and no rel="next". One request really is the whole ` +
      'list here.'];
  }

  const last = pageNumber(links.get('last'));
  if (last === null) {
    return ['more-pages-unsized',
      'rel="next" is present and rel="last" is not, so the total is only ' +
      'knowable by walking it. Terminate on the absence of rel="next", never ' +
      'on the absence of rel="last".'];
  }

  if (perPage === 1) {
    return ['more-pages',
      `${last} item(s) in total. A client that reads the first page and stops ` +
      `reports ${received}.`];
  }

  return ['more-pages',
    `${last} page(s) at per_page=${perPage}, so ${(last - 1) * perPage + 1} to ` +
    `${last * perPage} item(s) in total. A client that reads the first page and ` +
    `stops reports ${received}.`];
}

function arg(name, fallback) {
  const i = process.argv.indexOf(`--${name}`);
  return i === -1 ? fallback : process.argv[i + 1];
}

async function get(token, path, params = {}) {
  const url = new URL(API + path);
  for (const [k, v] of Object.entries(params)) url.searchParams.set(k, v);
  const res = await fetch(url, {
    headers: {
      Authorization: `Bearer ${token}`,
      Accept: 'application/vnd.github+json',
      'X-GitHub-Api-Version': '2022-11-28',
      'User-Agent': 'github-link-header-audit',
    },
  });
  if (res.status === 401) {
    throw new Error('401 from GitHub: GITHUB_TOKEN is missing, malformed or revoked');
  }
  if (res.status === 403) {
    throw new Error('403 from GitHub. If this is a rate limit, GET /rate_limit ' +
                    'reports the reset and does not itself consume quota');
  }
  if (res.status === 404) {
    throw new Error(`404 on ${path}: the repository does not exist, or this token ` +
                    'cannot see it');
  }
  if (!res.ok) throw new Error(`${res.status} from ${url.pathname}`);
  return res;
}

async function main() {
  const token = process.env.GITHUB_TOKEN;
  if (!token) {
    console.error('set GITHUB_TOKEN (a read-only token is enough)');
    process.exitCode = 2;
    return;
  }
  const repo = arg('repo');
  if (!repo) {
    console.error('usage: node github-link-header-audit.mjs --repo owner/name');
    process.exitCode = 2;
    return;
  }

  const probes = PROBES.map(([name, extra]) => [`/repos/${repo}/${name}`, extra]);

  let truncatable = 0;
  let remaining = '?';
  for (const [path, extra] of probes) {
    const res = await get(token, path, { per_page: 1, ...extra });
    remaining = res.headers.get('x-ratelimit-remaining') ?? '?';
    const body = await res.json();
    const received = Array.isArray(body) ? body.length : 0;
    const [state, detail] = verdict(parseLink(res.headers.get('link')), received, 1);

    const line = `${state.padEnd(18)} ${path}  ${detail}`;
    if (state === 'single-page') { console.log(line); continue; }
    truncatable += 1;
    console.warn(line);
    console.warn('  repair: follow rel="next" until it is absent -- ' +
                 'octokit.paginate() in Octokit, the PaginatedList in PyGithub, ' +
                 'gh api --paginate on the command line. Never build page URLs ' +
                 'by hand.');
  }

  console.log(`${probes.length} endpoint(s) probed, ${truncatable} with pages ` +
              `beyond the first; x-ratelimit-remaining ${remaining}`);
  process.exitCode = truncatable ? 1 : 0;
}

// Only run when invoked directly. The test file imports this module, and without
// the guard main() would run there too, fail on the missing token and set a
// non-zero exit code that fails the whole test file even as every test passes.
if (import.meta.url === `file://${process.argv[1]}`) {
  main().catch((err) => { console.error(err.message); process.exitCode = 2; });
}

from github_link_header_audit import page_number, parse_link, verdict

FULL = ('<https://api.github.com/repositories/1/pulls?per_page=1&page=2>; rel="next", '
        '<https://api.github.com/repositories/1/pulls?per_page=1&page=340>; rel="last"')


def test_link_header_parses_both_relations():
    links = parse_link(FULL)
    assert set(links) == {"next", "last"}
    assert page_number(links["last"]) == 340


def test_a_comma_inside_a_url_does_not_become_a_second_link():
    # labels=bug,ci is ordinary. Splitting the header on "," makes four broken
    # entries out of two good ones and the walk then terminates on page one.
    header = ('<https://api.github.com/repos/o/n/issues?labels=bug,ci&page=2>; rel="next", '
              '<https://api.github.com/repos/o/n/issues?labels=bug,ci&page=9>; rel="last"')
    links = parse_link(header)
    assert set(links) == {"next", "last"}
    assert links["next"].endswith("labels=bug,ci&page=2")


def test_no_link_header_is_a_single_page():
    state, detail = verdict(parse_link(None), 7, 1)
    assert state == "single-page"
    assert "7 item(s)" in detail


def test_rel_last_at_per_page_one_is_the_exact_count():
    state, detail = verdict(parse_link(FULL), 1, 1)
    assert state == "more-pages"
    assert "340 item(s)" in detail


def test_next_without_last_is_its_own_state():
    header = '<https://api.github.com/repos/o/n/branches?page=2>; rel="next"'
    state, detail = verdict(parse_link(header), 1, 1)
    assert state == "more-pages-unsized"
    assert 'rel="last"' in detail


def test_page_number_is_none_when_there_is_no_page_parameter():
    assert page_number("https://api.github.com/repos/o/n/pulls?per_page=100") is None
    assert page_number(None) is None

import { test } from 'node:test';
import assert from 'node:assert/strict';
import { pageNumber, parseLink, verdict } from './github-link-header-audit.mjs';

const FULL =
  '<https://api.github.com/repositories/1/pulls?per_page=1&page=2>; rel="next", ' +
  '<https://api.github.com/repositories/1/pulls?per_page=1&page=340>; rel="last"';

test('link header parses both relations', () => {
  const links = parseLink(FULL);
  assert.deepEqual([...links.keys()].sort(), ['last', 'next']);
  assert.equal(pageNumber(links.get('last')), 340);
});

test('a comma inside a url does not become a second link', () => {
  const header =
    '<https://api.github.com/repos/o/n/issues?labels=bug,ci&page=2>; rel="next", ' +
    '<https://api.github.com/repos/o/n/issues?labels=bug,ci&page=9>; rel="last"';
  const links = parseLink(header);
  assert.deepEqual([...links.keys()].sort(), ['last', 'next']);
  assert.match(links.get('next'), /labels=bug,ci&page=2$/);
});

test('no link header is a single page', () => {
  const [state, detail] = verdict(parseLink(null), 7, 1);
  assert.equal(state, 'single-page');
  assert.match(detail, /7 item\(s\)/);
});

test('rel=last at per_page=1 is the exact count', () => {
  const [state, detail] = verdict(parseLink(FULL), 1, 1);
  assert.equal(state, 'more-pages');
  assert.match(detail, /340 item\(s\)/);
});

test('next without last is its own state', () => {
  const header = '<https://api.github.com/repos/o/n/branches?page=2>; rel="next"';
  const [state, detail] = verdict(parseLink(header), 1, 1);
  assert.equal(state, 'more-pages-unsized');
  assert.match(detail, /rel="last"/);
});

test('page number is null when there is no page parameter', () => {
  assert.equal(pageNumber('https://api.github.com/repos/o/n/pulls?per_page=100'), null);
  assert.equal(pageNumber(null), null);
});