the installation covers only some repositories, silently

"""Report how much of an organization a GitHub App installation can actually see.

Read only. Two GET requests and no writes: an installation token plus a token
that can read the organization is enough. The repair is printed, never performed.
"""
import argparse
import logging
import os
import sys

import requests

logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s")
log = logging.getLogger("github_app_coverage_audit")

API = "https://api.github.com"
UA = "github-app-coverage-audit/1.0"


def expected_total(org):
    """Repositories the organization actually has, or None if it cannot be known.

    public_repos plus total_private_repos. total_private_repos is only returned
    to callers with enough access; when it is absent the public count is a floor
    and not a total. Returning it anyway would produce a coverage figure that
    understates the gap, so this returns None and lets the caller say so.
    """
    if not isinstance(org, dict):
        return None
    public = org.get("public_repos")
    private = org.get("total_private_repos")
    if public is None or private is None:
        return None
    return int(public) + int(private)


def coverage(selection, seen, expected):
    """Compare what the installation sees against what exists. Pure.

    Returns (state, detail). A `selected` installation whose count happens to
    match today is deliberately not the same state as `all`: it is correct now
    and nothing keeps it correct.
    """
    sel = str(selection or "").strip().lower()

    if sel == "all":
        return ("all-repositories",
                "%d repository(ies) visible, and repository_selection is 'all', "
                "so repositories created later join the installation "
                "automatically." % (seen,))

    if sel != "selected":
        return ("unknown-selection",
                "repository_selection is %r, which is neither 'all' nor "
                "'selected'. Do not assume coverage from a value you cannot "
                "interpret." % (selection,))

    if expected is None:
        return ("unmeasured",
                "%d repository(ies) selected. The organization's own total is "
                "not readable with this credential, so this is a count and not a "
                "coverage figure. Say so in the report rather than implying "
                "completeness." % (seen,))

    if seen > expected:
        return ("inconsistent",
                "%d repository(ies) visible against an organization total of %d. "
                "The installation spans more than this organization, or one of "
                "the two counts is stale. Resolve it before quoting either."
                % (seen, expected))

    if seen == expected:
        return ("selected-complete",
                "%d of %d today, and nothing keeps it that way: a 'selected' "
                "installation does not pick up repositories created later, so "
                "this is complete by coincidence." % (seen, expected))

    return ("partial",
            "%d of %d repositories. Every endpoint answers truthfully about "
            "those %d and says nothing at all about the other %d, so a clean "
            "report here covers %.0f%% of the organization."
            % (seen, expected, seen, expected - seen, 100.0 * seen / expected))


def get(session, url, **params):
    return session.get(url, params=params, timeout=30)


def installation_view(session, api):
    """repository_selection, total_count and the full names, from inside the App."""
    names = []
    selection, total = None, 0
    page = 1
    while True:
        r = get(session, api + "/installation/repositories", per_page=100, page=page)
        if r.status_code != 200:
            raise SystemExit("%d from GET /installation/repositories: this needs "
                             "an App installation token" % (r.status_code,))
        body = r.json()
        if page == 1:
            selection = body.get("repository_selection")
            total = int(body.get("total_count") or 0)
        items = body.get("repositories", [])
        names.extend(str(r_.get("full_name") or "") for r_ in items)
        if len(items) < 100:
            break
        page += 1
    return selection, total, names


def org_repo_names(session, api, org):
    """Every repository in the organization, from outside the installation."""
    names = []
    page = 1
    while True:
        r = get(session, "%s/orgs/%s/repos" % (api, org), per_page=100, page=page)
        if r.status_code != 200:
            return None
        items = r.json()
        names.extend(str(x.get("full_name") or "") for x in items)
        if len(items) < 100:
            break
        page += 1
    return names


def main():
    ap = argparse.ArgumentParser(description=__doc__)
    ap.add_argument("--org", required=True,
                    help="the organization the installation is meant to cover")
    ap.add_argument("--api", default=API,
                    help="API host, for GitHub Enterprise Server")
    ap.add_argument("--list-missing", action="store_true",
                    help="name the repositories outside the installation")
    args = ap.parse_args()

    token = os.environ.get("GITHUB_TOKEN")
    if not token:
        log.error("set GITHUB_TOKEN (an App installation token, read-only)")
        return 2

    session = requests.Session()
    session.headers.update({
        "Authorization": "Bearer " + token,
        "Accept": "application/vnd.github+json",
        "X-GitHub-Api-Version": "2022-11-28",
        "User-Agent": UA,
    })

    selection, seen, inside = installation_view(session, args.api)

    org_response = get(session, "%s/orgs/%s" % (args.api, args.org))
    expected = expected_total(org_response.json()) if org_response.status_code == 200 else None

    state, detail = coverage(selection, seen, expected)
    line = "%-18s %s" % (state, detail)
    if state == "all-repositories":
        log.info(line)
        return 0

    log.warning(line)
    if args.list_missing:
        outside = org_repo_names(session, args.api, args.org)
        if outside is None:
            log.warning("  the organization's repository list is not readable "
                        "with this credential, so the missing names cannot be "
                        "printed. The counts above still stand.")
        else:
            have = {n.lower() for n in inside}
            missing = sorted(n for n in outside if n.lower() not in have)
            for name in missing[:50]:
                log.warning("  outside the installation: %s", name)
            if len(missing) > 50:
                log.warning("  ... and %d more", len(missing) - 50)

    log.warning("  repair: switch the installation to All repositories, or add "
                "the missing repositories to it. Then have the tool print its "
                "own coverage next to its findings, so a clean report can never "
                "again appear without the number of repositories behind it.")
    return 1


if __name__ == "__main__":
    sys.exit(main())

/**
 * Report how much of an organization a GitHub App installation can actually see.
 *
 * Read only. Two GET requests and no writes. The repair is printed, never
 * performed.
 */
const API = 'https://api.github.com';
const UA = 'github-app-coverage-audit/1.0';

/**
 * Repositories the organization actually has, or null if it cannot be known.
 * total_private_repos is only returned to callers with enough access; without it
 * the public count is a floor, and a coverage figure built on a floor understates
 * the gap, so this returns null rather than a number.
 */
export function expectedTotal(org) {
  if (!org || typeof org !== 'object') return null;
  const pub = org.public_repos;
  const priv = org.total_private_repos;
  if (pub === null || pub === undefined) return null;
  if (priv === null || priv === undefined) return null;
  return Number(pub) + Number(priv);
}

/**
 * Compare what the installation sees against what exists. Pure.
 * Returns [state, detail].
 */
export function coverage(selection, seen, expected) {
  const sel = String(selection ?? '').trim().toLowerCase();

  if (sel === 'all') {
    return ['all-repositories',
      `${seen} repository(ies) visible, and repository_selection is 'all', so ` +
      'repositories created later join the installation automatically.'];
  }

  if (sel !== 'selected') {
    return ['unknown-selection',
      `repository_selection is ${JSON.stringify(selection)}, which is neither ` +
      "'all' nor 'selected'. Do not assume coverage from a value you cannot " +
      'interpret.'];
  }

  if (expected === null || expected === undefined) {
    return ['unmeasured',
      `${seen} repository(ies) selected. The organization's own total is not ` +
      'readable with this credential, so this is a count and not a coverage ' +
      'figure. Say so in the report rather than implying completeness.'];
  }

  if (seen > expected) {
    return ['inconsistent',
      `${seen} repository(ies) visible against an organization total of ` +
      `${expected}. The installation spans more than this organization, or one ` +
      'of the two counts is stale. Resolve it before quoting either.'];
  }

  if (seen === expected) {
    return ['selected-complete',
      `${seen} of ${expected} today, and nothing keeps it that way: a ` +
      "'selected' installation does not pick up repositories created later, so " +
      'this is complete by coincidence.'];
  }

  const pct = Math.round((100 * seen) / expected);
  return ['partial',
    `${seen} of ${expected} repositories. Every endpoint answers truthfully ` +
    `about those ${seen} and says nothing at all about the other ` +
    `${expected - seen}, so a clean report here covers ${pct}% of the ` +
    'organization.'];
}

function headers(token) {
  return {
    Authorization: `Bearer ${token}`,
    Accept: 'application/vnd.github+json',
    'X-GitHub-Api-Version': '2022-11-28',
    'User-Agent': UA,
  };
}

async function get(token, url, params = {}) {
  const u = new URL(url);
  for (const [k, v] of Object.entries(params)) u.searchParams.set(k, v);
  return fetch(u, { headers: headers(token) });
}

export async function installationView(token, api = API) {
  const names = [];
  let selection = null;
  let total = 0;
  let page = 1;
  for (;;) {
    const res = await get(token, `${api}/installation/repositories`,
                          { per_page: 100, page });
    if (res.status !== 200) {
      throw new Error(`${res.status} from GET /installation/repositories: this ` +
                      'needs an App installation token');
    }
    const body = await res.json();
    if (page === 1) {
      selection = body.repository_selection;
      total = Number(body.total_count ?? 0);
    }
    const items = body.repositories ?? [];
    names.push(...items.map((r) => String(r.full_name ?? '')));
    if (items.length < 100) break;
    page += 1;
  }
  return { selection, total, names };
}

async function main() {
  const token = process.env.GITHUB_TOKEN;
  if (!token) {
    console.error('set GITHUB_TOKEN (an App installation token, read-only)');
    process.exitCode = 2;
    return;
  }
  const at = process.argv.indexOf('--org');
  const org = at >= 0 ? process.argv[at + 1] : null;
  if (!org) {
    console.error('pass --org <login>');
    process.exitCode = 2;
    return;
  }

  const { selection, total: seen } = await installationView(token);

  const orgRes = await get(token, `${API}/orgs/${org}`);
  const expected = orgRes.status === 200 ? expectedTotal(await orgRes.json()) : null;

  const [state, detail] = coverage(selection, seen, expected);
  const line = `${state.padEnd(18)} ${detail}`;
  if (state === 'all-repositories') {
    console.log(line);
    return;
  }

  console.warn(line);
  console.warn('  repair: switch the installation to All repositories, or add the ' +
               'missing repositories to it. Then have the tool print its own ' +
               'coverage next to its findings, so a clean report can never again ' +
               'appear without the number of repositories behind it.');
  process.exitCode = 1;
}

// Only run when invoked directly. The test file imports this module, and without
// the guard main() would run there too, fail on the missing token, and set a
// non-zero exit code that fails the whole test file even as every test passes.
if (import.meta.url === `file://${process.argv[1]}`) {
  main().catch((err) => { console.error(err.message); process.exitCode = 2; });
}

from github_app_coverage_audit import coverage, expected_total


def test_org_total_needs_both_halves():
    assert expected_total({"public_repos": 40, "total_private_repos": 100}) == 140
    assert expected_total({"public_repos": 0, "total_private_repos": 0}) == 0


def test_a_missing_private_count_yields_no_total_at_all():
    # Falling back to public_repos here is how an unmeasurable gap becomes a
    # reassuring percentage.
    assert expected_total({"public_repos": 40}) is None
    assert expected_total({}) is None
    assert expected_total(None) is None


def test_all_repositories_is_the_only_good_news():
    state, detail = coverage("all", 140, 140)
    assert state == "all-repositories"
    assert "automatically" in detail


def test_twelve_of_a_hundred_and_forty_names_the_gap_and_the_share():
    state, detail = coverage("selected", 12, 140)
    assert state == "partial"
    assert "12 of 140" in detail
    assert "128" in detail
    assert "9%" in detail


def test_selected_and_complete_is_not_the_same_as_all():
    # Correct today, and nothing keeps it correct.
    state, detail = coverage("selected", 140, 140)
    assert state == "selected-complete"
    assert "coincidence" in detail


def test_no_org_total_means_a_count_not_a_coverage_figure():
    state, detail = coverage("selected", 12, None)
    assert state == "unmeasured"
    assert "not a coverage figure" in detail


def test_seeing_more_than_exists_is_reported_rather_than_averaged_away():
    state, _ = coverage("selected", 150, 140)
    assert state == "inconsistent"


def test_an_uninterpretable_selection_is_never_assumed_complete():
    assert coverage(None, 12, 140)[0] == "unknown-selection"
    assert coverage("some-new-value", 12, 140)[0] == "unknown-selection"

import { test } from 'node:test';
import assert from 'node:assert/strict';
import { coverage, expectedTotal } from './github-app-coverage-audit.mjs';

test('org total needs both halves', () => {
  assert.equal(expectedTotal({ public_repos: 40, total_private_repos: 100 }), 140);
  assert.equal(expectedTotal({ public_repos: 0, total_private_repos: 0 }), 0);
});

test('a missing private count yields no total at all', () => {
  assert.equal(expectedTotal({ public_repos: 40 }), null);
  assert.equal(expectedTotal({}), null);
  assert.equal(expectedTotal(null), null);
});

test('all repositories is the only good news', () => {
  const [state, detail] = coverage('all', 140, 140);
  assert.equal(state, 'all-repositories');
  assert.match(detail, /automatically/);
});

test('twelve of a hundred and forty names the gap and the share', () => {
  const [state, detail] = coverage('selected', 12, 140);
  assert.equal(state, 'partial');
  assert.match(detail, /12 of 140/);
  assert.match(detail, /128/);
  assert.match(detail, /9%/);
});

test('selected and complete is not the same as all', () => {
  const [state, detail] = coverage('selected', 140, 140);
  assert.equal(state, 'selected-complete');
  assert.match(detail, /coincidence/);
});

test('no org total means a count, not a coverage figure', () => {
  const [state, detail] = coverage('selected', 12, null);
  assert.equal(state, 'unmeasured');
  assert.match(detail, /not a coverage figure/);
});

test('seeing more than exists is reported rather than averaged away', () => {
  assert.equal(coverage('selected', 150, 140)[0], 'inconsistent');
});

test('an uninterpretable selection is never assumed complete', () => {
  assert.equal(coverage(null, 12, 140)[0], 'unknown-selection');
  assert.equal(coverage('some-new-value', 12, 140)[0], 'unknown-selection');
});