search returns at most 1,000 results whatever total_count says

"""Report search queries whose results cannot be paged through in full.

Read only. GET requests and nothing else: a token with read access is enough.
The repair is printed, never performed.

The cap is a property of the query, so this is one of the few checks here that
gives a complete answer: total_count above 1,000 means results exist that no
client, correct or otherwise, can reach.
"""
import argparse
import logging
import os
import sys

import requests

logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s")
log = logging.getLogger("github_search_cap_audit")

API = "https://api.github.com"

CAP = 1000
NEAR = 900
MAX_PER_PAGE = 100


def last_reachable_page(per_page=MAX_PER_PAGE):
    """The highest page number that lies entirely inside the 1,000-result cap.

    Pure. The request that straddles the boundary is the one that returns 422,
    so this is the page after which a walk stops working: 10 at per_page=100,
    33 at per_page=30.
    """
    size = min(max(int(per_page or 30), 1), MAX_PER_PAGE)
    return CAP // size


def reach(total_count, per_page=MAX_PER_PAGE):
    """Classify one query against the cap. Pure. Returns (state, detail)."""
    total = int(total_count or 0)
    last = last_reachable_page(per_page)

    if total <= 0:
        return ("no-matches", "no results; the query matches nothing")

    if total > CAP:
        slices = -(-total // CAP)
        return ("capped",
                "total_count is %d and only the first %d are reachable, so %d "
                "match(es) cannot be paged to at any page size. Page %d at "
                "per_page=%d is the last that works; the next one returns 422. "
                "Partition into at least %d narrower queries."
                % (total, CAP, total - CAP, last, per_page, slices))

    if total >= NEAR:
        return ("near-cap",
                "total_count is %d, inside the 1,000-result cap but close to it. "
                "This query starts losing results silently as soon as it grows "
                "past %d; partition it now rather than after."
                % (total, CAP))

    return ("reachable",
            "total_count is %d, all reachable in %d request(s) at per_page=%d."
            % (total, -(-total // min(max(int(per_page), 1), MAX_PER_PAGE)), per_page))


def get(session, path, **params):
    r = session.get(API + path, params=params, timeout=30)
    if r.status_code == 401:
        raise SystemExit("401 from GitHub: GITHUB_TOKEN is missing, malformed or "
                         "revoked")
    if r.status_code == 403:
        raise SystemExit("403 from GitHub. Search has its own small per-minute "
                         "bucket; GET /rate_limit reports resources.search and "
                         "does not itself consume quota")
    if r.status_code == 422:
        raise SystemExit("422 from search: either the query is malformed or it "
                         "already reaches past the 1,000-result cap")
    r.raise_for_status()
    return r.json()


def main():
    ap = argparse.ArgumentParser(description=__doc__)
    ap.add_argument("--query", action="append", required=True,
                    help="a search query string. Repeatable, so a partitioned "
                         "query can be checked slice by slice.")
    ap.add_argument("--endpoint", default="issues",
                    choices=["issues", "repositories", "commits", "code", "users",
                             "labels", "topics"],
                    help="which /search/ endpoint to ask")
    ap.add_argument("--per-page", type=int, default=MAX_PER_PAGE,
                    help="the page size your client sends, used for the page "
                         "arithmetic")
    args = ap.parse_args()

    token = os.environ.get("GITHUB_TOKEN")
    if not token:
        log.error("set GITHUB_TOKEN (a read-only token is enough)")
        return 2

    session = requests.Session()
    session.headers.update({
        "Authorization": "Bearer " + token,
        "Accept": "application/vnd.github+json",
        "X-GitHub-Api-Version": "2022-11-28",
        "User-Agent": "github-search-cap-audit",
    })

    # Free to ask: /rate_limit is not billed against any bucket, and search is
    # not billed against core, so this is the only cheap way to see the bucket
    # these queries will actually spend.
    quota = get(session, "/rate_limit").get("resources", {}).get("search", {})
    log.info("search bucket: %s of %s remaining, resets at %s",
             quota.get("remaining", "?"), quota.get("limit", "?"),
             quota.get("reset", "?"))

    over = 0
    for q in args.query:
        # per_page=1 is enough: total_count is on every page, and the first item
        # costs less to transfer than a hundred you are not going to read.
        body = get(session, "/search/%s" % args.endpoint, q=q, per_page=1)
        state, detail = reach(body.get("total_count"), args.per_page)
        line = "%-10s %s  %s" % (state, q, detail)
        if state in ("capped", "near-cap"):
            over += 1
            log.warning(line)
            log.warning("  repair: split this query by created: date ranges, by "
                        "repo:, or by label until every slice reports under "
                        "1,000, then union the slices and de-duplicate on id. "
                        "For a full inventory use the matching list endpoint "
                        "instead, which has no such cap.")
        else:
            log.info(line)

    log.info("%d quer(y/ies), %d over or near the %d-result cap",
             len(args.query), over, CAP)
    return 1 if over else 0


if __name__ == "__main__":
    sys.exit(main())

/**
 * Report search queries whose results cannot be paged through in full.
 *
 * Read only. GET requests and nothing else: a token with read access is enough.
 * The repair is printed, never performed.
 */
const API = 'https://api.github.com';

const CAP = 1000;
const NEAR = 900;
const MAX_PER_PAGE = 100;

/**
 * The highest page number that lies entirely inside the 1,000-result cap. Pure.
 * The request that straddles the boundary is the one that returns 422.
 */
export function lastReachablePage(perPage = MAX_PER_PAGE) {
  const size = Math.min(Math.max(Number(perPage) || 30, 1), MAX_PER_PAGE);
  return Math.floor(CAP / size);
}

/** Classify one query against the cap. Pure. Returns [state, detail]. */
export function reach(totalCount, perPage = MAX_PER_PAGE) {
  const total = Number(totalCount) || 0;
  const size = Math.min(Math.max(Number(perPage) || 30, 1), MAX_PER_PAGE);
  const last = lastReachablePage(perPage);

  if (total <= 0) return ['no-matches', 'no results; the query matches nothing'];

  if (total > CAP) {
    const slices = Math.ceil(total / CAP);
    return ['capped',
      `total_count is ${total} and only the first ${CAP} are reachable, so ` +
      `${total - CAP} match(es) cannot be paged to at any page size. Page ` +
      `${last} at per_page=${perPage} is the last that works; the next one ` +
      `returns 422. Partition into at least ${slices} narrower queries.`];
  }

  if (total >= NEAR) {
    return ['near-cap',
      `total_count is ${total}, inside the 1,000-result cap but close to it. ` +
      `This query starts losing results silently as soon as it grows past ` +
      `${CAP}; partition it now rather than after.`];
  }

  return ['reachable',
    `total_count is ${total}, all reachable in ${Math.ceil(total / size)} ` +
    `request(s) at per_page=${perPage}.`];
}

function args(name) {
  const out = [];
  process.argv.forEach((a, i) => { if (a === `--${name}`) out.push(process.argv[i + 1]); });
  return out;
}

async function get(token, path, params = {}) {
  const url = new URL(API + path);
  for (const [k, v] of Object.entries(params)) url.searchParams.set(k, v);
  const res = await fetch(url, {
    headers: {
      Authorization: `Bearer ${token}`,
      Accept: 'application/vnd.github+json',
      'X-GitHub-Api-Version': '2022-11-28',
      'User-Agent': 'github-search-cap-audit',
    },
  });
  if (res.status === 401) {
    throw new Error('401 from GitHub: GITHUB_TOKEN is missing, malformed or revoked');
  }
  if (res.status === 403) {
    throw new Error('403 from GitHub. Search has its own small per-minute bucket; ' +
                    'GET /rate_limit reports resources.search and does not itself ' +
                    'consume quota');
  }
  if (res.status === 422) {
    throw new Error('422 from search: the query is malformed, or it already ' +
                    'reaches past the 1,000-result cap');
  }
  if (!res.ok) throw new Error(`${res.status} from ${url.pathname}`);
  return res.json();
}

async function main() {
  const token = process.env.GITHUB_TOKEN;
  const queries = args('query');
  if (!token || queries.length === 0) {
    console.error('set GITHUB_TOKEN and pass --query "..." at least once');
    process.exitCode = 2;
    return;
  }
  const endpoint = args('endpoint')[0] ?? 'issues';
  const perPage = Number(args('per-page')[0] ?? MAX_PER_PAGE) || MAX_PER_PAGE;

  // Free to ask: /rate_limit is not billed against any bucket, and search is not
  // billed against core.
  const quota = (await get(token, '/rate_limit')).resources?.search ?? {};
  console.log(`search bucket: ${quota.remaining ?? '?'} of ${quota.limit ?? '?'} ` +
              `remaining, resets at ${quota.reset ?? '?'}`);

  let over = 0;
  for (const q of queries) {
    const body = await get(token, `/search/${endpoint}`, { q, per_page: 1 });
    const [state, detail] = reach(body.total_count, perPage);
    const line = `${state.padEnd(10)} ${q}  ${detail}`;
    if (state === 'capped' || state === 'near-cap') {
      over += 1;
      console.warn(line);
      console.warn('  repair: split this query by created: date ranges, by repo:, ' +
                   'or by label until every slice reports under 1,000, then union ' +
                   'the slices and de-duplicate on id. For a full inventory use ' +
                   'the matching list endpoint instead, which has no such cap.');
    } else {
      console.log(line);
    }
  }

  console.log(`${queries.length} quer(y/ies), ${over} over or near the ${CAP}-result cap`);
  process.exitCode = over ? 1 : 0;
}

// Only run when invoked directly, so the test file can import the pure functions
// without main() running, failing on the missing token and failing the suite.
if (import.meta.url === `file://${process.argv[1]}`) {
  main().catch((err) => { console.error(err.message); process.exitCode = 2; });
}

from github_search_cap_audit import last_reachable_page, reach


def test_a_small_query_is_fully_reachable():
    state, detail = reach(240, 100)
    assert state == "reachable"
    assert "3 request(s)" in detail


def test_a_query_over_the_cap_names_what_is_unreachable():
    state, detail = reach(24831, 100)
    assert state == "capped"
    assert "23831 match(es)" in detail
    assert "at least 25 narrower queries" in detail


def test_just_under_the_cap_is_a_warning_not_a_pass():
    # 950 works today and silently loses results the moment it passes 1,000.
    state, detail = reach(950, 100)
    assert state == "near-cap"
    assert "950" in detail


def test_no_matches_is_not_confused_with_a_capped_query():
    assert reach(0, 100)[0] == "no-matches"
    assert reach(None, 100)[0] == "no-matches"


def test_the_last_working_page_depends_on_the_page_size():
    assert last_reachable_page(100) == 10
    assert last_reachable_page(30) == 33
    assert last_reachable_page(1) == 1000


def test_page_size_above_the_maximum_is_clamped_before_the_arithmetic():
    assert last_reachable_page(500) == 10

import { test } from 'node:test';
import assert from 'node:assert/strict';
import { lastReachablePage, reach } from './github-search-cap-audit.mjs';

test('a small query is fully reachable', () => {
  const [state, detail] = reach(240, 100);
  assert.equal(state, 'reachable');
  assert.match(detail, /3 request\(s\)/);
});

test('a query over the cap names what is unreachable', () => {
  const [state, detail] = reach(24831, 100);
  assert.equal(state, 'capped');
  assert.match(detail, /23831 match\(es\)/);
  assert.match(detail, /at least 25 narrower queries/);
});

test('just under the cap is a warning, not a pass', () => {
  const [state, detail] = reach(950, 100);
  assert.equal(state, 'near-cap');
  assert.match(detail, /950/);
});

test('no matches is not confused with a capped query', () => {
  assert.equal(reach(0, 100)[0], 'no-matches');
  assert.equal(reach(null, 100)[0], 'no-matches');
});

test('the last working page depends on the page size', () => {
  assert.equal(lastReachablePage(100), 10);
  assert.equal(lastReachablePage(30), 33);
  assert.equal(lastReachablePage(1), 1000);
});

test('page size above the maximum is clamped before the arithmetic', () => {
  assert.equal(lastReachablePage(500), 10);
});