the installation is suspended and every call it makes 403s

"""Say whether a GitHub App installation is suspended, and stop retrying if so.

Read only. GETs against the App's own installation records with the App JWT,
plus one optional probe with an installation access token. Nothing is minted,
unsuspended or changed. There is an endpoint that unsuspends an installation
and it is a write, so this script does not call it; it prints the request you
have to make of an organization owner instead.

An owner can suspend an installation rather than removing it. The record
survives, so the App still lists it and a stored id still resolves, but tokens
minted for it are refused and webhook delivery stops. The only clean signal is
the suspended_at field on the installation record, which is readable with the
App's JWT and not with a token minted from it.

Environment:

    GITHUB_APP_JWT              the JWT your own signing code produced
    GITHUB_INSTALLATION_TOKEN   optional, used only to corroborate
"""
import argparse
import json
import logging
import os
import sys
from datetime import datetime, timezone

import requests

logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s")
log = logging.getLogger("github_installation_suspension")

API = "https://api.github.com"
UA = "github-installation-suspension/1.0"

# States that no amount of waiting will clear. Reporting this is the point of
# the script: a suspended installation fails identically forever, so a backoff
# loop around it is a way of spending quota on a decision somebody already made.
TERMINAL = ("suspended", "not-listed")


def suspended_at(inst):
    """The suspension timestamp on an installation record, or None. Pure.

    Tolerant on purpose. Depending on which layer deserialised the JSON the
    absent case arrives as a missing key, as None, as an empty string or as the
    four characters n-u-l-l, and treating any of those as a timestamp would
    report a healthy installation as suspended.
    """
    if not isinstance(inst, dict):
        return None
    raw = inst.get("suspended_at")
    if raw is None:
        return None
    text = str(raw).strip()
    if not text or text.lower() in ("null", "none"):
        return None
    return text


def is_suspended(inst):
    """Whether an installation record carries a suspension. Pure."""
    return suspended_at(inst) is not None


def suspended_by(inst):
    """Who suspended it, where the record says. Pure.

    suspended_by is a user object when it is present at all, and it can be
    absent on a record that is genuinely suspended, so an unknown actor is
    never taken as evidence that nothing happened.
    """
    if not isinstance(inst, dict):
        return None
    who = inst.get("suspended_by")
    if isinstance(who, dict):
        login = who.get("login")
        return str(login) if login else None
    if isinstance(who, str) and who.strip():
        return who.strip()
    return None


def parsed_time(text):
    """An ISO 8601 timestamp as an aware datetime, or None. Pure."""
    raw = str(text or "").strip()
    if not raw:
        return None
    if raw.endswith("Z"):
        raw = raw[:-1] + "+00:00"
    try:
        when = datetime.fromisoformat(raw)
    except ValueError:
        return None
    return when if when.tzinfo else when.replace(tzinfo=timezone.utc)


def days_since(text, now):
    """Whole days between a timestamp and now, or None. Pure."""
    when = parsed_time(text)
    if when is None or now is None:
        return None
    return (now - when).days


def account_of(inst):
    """The login of the account an installation sits on. Pure."""
    if not isinstance(inst, dict):
        return "an unnamed account"
    account = inst.get("account")
    if isinstance(account, dict) and account.get("login"):
        return str(account["login"])
    return "an unnamed account"


def find(installations, installation_id):
    """The record for one installation id, or None. Pure.

    The id is compared as text because it reaches this script from a config
    file, an environment variable or a command line, and 41234567 and the
    string "41234567" are the same installation.
    """
    if installation_id is None:
        return None
    wanted = str(installation_id).strip()
    for inst in installations or []:
        if isinstance(inst, dict) and str(inst.get("id", "")).strip() == wanted:
            return inst
    return None


def summarize(installations):
    """Counts across every installation the App can see. Pure."""
    rows = [i for i in (installations or []) if isinstance(i, dict)]
    suspended = [i for i in rows if is_suspended(i)]
    return {"total": len(rows), "suspended": len(suspended),
            "active": len(rows) - len(suspended),
            "suspended_ids": [i.get("id") for i in suspended]}


def verdict(target, probe_status=None, now=None):
    """Turn one installation record and an optional probe into a finding. Pure.

    target is the record for the installation being asked about, or None when
    the id is not in the list at all. probe_status is what an installation
    access token got from GET /installation/repositories, where one was
    available; it corroborates and never decides, because a 403 on its own is
    the least specific thing the GitHub API says.
    """
    if target is None:
        return ("not-listed",
                "this installation id is not among the ones the App can see. "
                "Suspension keeps the record, so an absent id means the App "
                "was removed and possibly reinstalled under a new id, which "
                "is a different repair.")
    ident = "installation %s on %s" % (target.get("id", "?"), account_of(target))
    when = suspended_at(target)
    if when is not None:
        age = days_since(when, now)
        who = suspended_by(target)
        return ("suspended",
                "%s was suspended at %s%s%s. Every token minted for it is "
                "refused and webhook delivery has stopped. Retrying cannot "
                "clear this." % (ident, when,
                                 " by %s" % who if who else "",
                                 ", %d day(s) ago" % age if age is not None else ""))
    if probe_status in (401, 403):
        return ("active-but-refused",
                "%s is listed and not suspended, yet an installation token "
                "got %d. The refusal is about a permission, a route or the "
                "token itself rather than about suspension."
                % (ident, probe_status))
    return ("active", "%s is listed and not suspended." % ident)


def retryable(state):
    """Whether a caller should ever try this installation again. Pure."""
    return state not in TERMINAL


def repair(state, target):
    """The sentence a reader has to act on. Pure."""
    if state == "suspended":
        return ("an organization owner unsuspends it from the %s account's "
                "Installed GitHub Apps page. Retrying cannot help: stop the "
                "queue for this installation and alert once."
                % account_of(target))
    if state == "not-listed":
        return ("resolve the installation id at runtime from the org's own "
                "installation record, or from the installation.id field on an "
                "incoming webhook, rather than storing it.")
    if state == "active-but-refused":
        return ("read the accepted-permissions header on the failing response "
                "and diff it against the permissions this installation "
                "granted. Suspension is not the cause here.")
    return "nothing. This installation is usable."


def get(session, path):
    """One GET. Returns (status, json-or-None)."""
    url = API + path if path.startswith("/") else path
    r = session.get(url, timeout=30)
    try:
        body = r.json()
    except ValueError:
        body = None
    return r.status_code, body


def list_installations(session, pages=10):
    """Every installation this App can see, paginated. Read only."""
    out = []
    for page in range(1, pages + 1):
        status, body = get(session, "/app/installations?per_page=100&page=%d" % page)
        if status != 200 or not isinstance(body, list):
            if page == 1:
                log.error("GET /app/installations returned %d; the JWT is the "
                          "credential this endpoint wants", status)
            break
        out.extend(body)
        if len(body) < 100:
            break
    return out


def main():
    ap = argparse.ArgumentParser(description=__doc__)
    ap.add_argument("--installation-id",
                    default=os.environ.get("GITHUB_INSTALLATION_ID"),
                    help="the installation to ask about; omit to report on all")
    args = ap.parse_args()

    jwt = os.environ.get("GITHUB_APP_JWT")
    if not jwt:
        log.error("set GITHUB_APP_JWT to the JWT your own signing code "
                  "produced. suspended_at lives on the installation record, "
                  "and installation records are read with the App's JWT")
        return 2

    session = requests.Session()
    session.headers.update({
        "Authorization": "Bearer " + jwt,
        "Accept": "application/vnd.github+json",
        "X-GitHub-Api-Version": "2022-11-28",
        "User-Agent": UA,
    })

    installations = list_installations(session)
    stats = summarize(installations)
    log.info("%d installation(s) visible to this App, %d suspended",
             stats["total"], stats["suspended"])

    probe_status = None
    token = os.environ.get("GITHUB_INSTALLATION_TOKEN")
    if token:
        probe = requests.Session()
        probe.headers.update({
            "Authorization": "Bearer " + token,
            "Accept": "application/vnd.github+json",
            "X-GitHub-Api-Version": "2022-11-28",
            "User-Agent": UA,
        })
        probe_status, _ = get(probe, "/installation/repositories?per_page=1")
        log.info("installation token: GET /installation/repositories returned %d",
                 probe_status)

    now = datetime.now(timezone.utc)
    findings = []
    if args.installation_id:
        target = find(installations, args.installation_id)
        state, detail = verdict(target, probe_status, now)
        findings.append({"installation_id": args.installation_id, "state": state,
                         "detail": detail, "retryable": retryable(state)})
        log.info("%s: %s", state, detail)
        log.info("repair: %s", repair(state, target))
    else:
        for inst in installations:
            state, detail = verdict(inst, None, now)
            findings.append({"installation_id": inst.get("id"), "state": state,
                             "detail": detail, "retryable": retryable(state)})
            if state != "active":
                log.info("%s: %s", state, detail)
                log.info("repair: %s", repair(state, inst))
        if stats["suspended"] == 0:
            log.info("active: no installation of this App is suspended")

    print(json.dumps({"visible": stats["total"], "suspended": stats["suspended"],
                      "suspended_ids": stats["suspended_ids"],
                      "probe_status": probe_status,
                      "findings": findings}, indent=2, default=str))
    return 1 if any(not f["retryable"] for f in findings) else 0


if __name__ == "__main__":
    sys.exit(main())

/**
 * Say whether a GitHub App installation is suspended, and stop retrying if so.
 *
 * Read only. GETs against the App's own installation records with the App JWT,
 * plus one optional probe with an installation access token. The endpoint that
 * unsuspends an installation is a write and is not called here; the script
 * prints the request you have to make of an organization owner instead.
 *
 * Environment:
 *   GITHUB_APP_JWT             the JWT your own signing code produced
 *   GITHUB_INSTALLATION_TOKEN  optional, used only to corroborate
 *   GITHUB_INSTALLATION_ID     optional, the installation to ask about
 */
const API = 'https://api.github.com';
const UA = 'github-installation-suspension/1.0';

/** States no amount of waiting will clear. */
export const TERMINAL = ['suspended', 'not-listed'];

/**
 * The suspension timestamp on an installation record, or null. Pure.
 * Absent arrives as a missing key, null, an empty string or the four
 * characters n-u-l-l depending on what deserialised the JSON.
 */
export function suspendedAt(inst) {
  if (!inst || typeof inst !== 'object') return null;
  const raw = inst.suspended_at;
  if (raw === null || raw === undefined) return null;
  const text = String(raw).trim();
  if (!text || ['null', 'none'].includes(text.toLowerCase())) return null;
  return text;
}

/** Whether an installation record carries a suspension. Pure. */
export function isSuspended(inst) {
  return suspendedAt(inst) !== null;
}

/** Who suspended it, where the record says. Pure. */
export function suspendedBy(inst) {
  if (!inst || typeof inst !== 'object') return null;
  const who = inst.suspended_by;
  if (who && typeof who === 'object') return who.login ? String(who.login) : null;
  if (typeof who === 'string' && who.trim()) return who.trim();
  return null;
}

/** An ISO 8601 timestamp as epoch milliseconds, or null. Pure. */
export function parsedTime(text) {
  const raw = String(text ?? '').trim();
  if (!raw) return null;
  const ms = Date.parse(raw);
  return Number.isNaN(ms) ? null : ms;
}

/** Whole days between a timestamp and now, or null. Pure. */
export function daysSince(text, nowMs) {
  const when = parsedTime(text);
  if (when === null || nowMs === null || nowMs === undefined) return null;
  return Math.floor((nowMs - when) / 86400000);
}

/** The login of the account an installation sits on. Pure. */
export function accountOf(inst) {
  if (!inst || typeof inst !== 'object') return 'an unnamed account';
  const account = inst.account;
  if (account && typeof account === 'object' && account.login) return String(account.login);
  return 'an unnamed account';
}

/** The record for one installation id, or null. Pure. Compared as text. */
export function find(installations, installationId) {
  if (installationId === null || installationId === undefined) return null;
  const wanted = String(installationId).trim();
  for (const inst of installations ?? []) {
    if (inst && typeof inst === 'object' && String(inst.id ?? '').trim() === wanted) return inst;
  }
  return null;
}

/** Counts across every installation the App can see. Pure. */
export function summarize(installations) {
  const rows = (installations ?? []).filter((i) => i && typeof i === 'object');
  const suspended = rows.filter(isSuspended);
  return {
    total: rows.length,
    suspended: suspended.length,
    active: rows.length - suspended.length,
    suspended_ids: suspended.map((i) => i.id),
  };
}

/** Turn one installation record and an optional probe into a finding. Pure. */
export function verdict(target, probeStatus = null, nowMs = null) {
  if (!target) {
    return ['not-listed',
      'this installation id is not among the ones the App can see. ' +
      'Suspension keeps the record, so an absent id means the App was ' +
      'removed and possibly reinstalled under a new id, which is a ' +
      'different repair.'];
  }
  const ident = `installation ${target.id ?? '?'} on ${accountOf(target)}`;
  const when = suspendedAt(target);
  if (when !== null) {
    const age = daysSince(when, nowMs);
    const who = suspendedBy(target);
    return ['suspended',
      `${ident} was suspended at ${when}${who ? ` by ${who}` : ''}` +
      `${age !== null ? `, ${age} day(s) ago` : ''}. Every token minted for ` +
      'it is refused and webhook delivery has stopped. Retrying cannot clear this.'];
  }
  if (probeStatus === 401 || probeStatus === 403) {
    return ['active-but-refused',
      `${ident} is listed and not suspended, yet an installation token got ` +
      `${probeStatus}. The refusal is about a permission, a route or the ` +
      'token itself rather than about suspension.'];
  }
  return ['active', `${ident} is listed and not suspended.`];
}

/** Whether a caller should ever try this installation again. Pure. */
export function retryable(state) {
  return !TERMINAL.includes(state);
}

/** The sentence a reader has to act on. Pure. */
export function repair(state, target) {
  if (state === 'suspended') {
    return `an organization owner unsuspends it from the ${accountOf(target)} ` +
      "account's Installed GitHub Apps page. Retrying cannot help: stop the " +
      'queue for this installation and alert once.';
  }
  if (state === 'not-listed') {
    return 'resolve the installation id at runtime from the org\'s own ' +
      'installation record, or from the installation.id field on an incoming ' +
      'webhook, rather than storing it.';
  }
  if (state === 'active-but-refused') {
    return 'read the accepted-permissions header on the failing response and ' +
      'diff it against the permissions this installation granted. Suspension ' +
      'is not the cause here.';
  }
  return 'nothing. This installation is usable.';
}

async function get(credential, path) {
  const url = path.startsWith('/') ? API + path : path;
  const res = await fetch(url, {
    headers: {
      Authorization: `Bearer ${credential}`,
      Accept: 'application/vnd.github+json',
      'X-GitHub-Api-Version': '2022-11-28',
      'User-Agent': UA,
    },
  });
  let body = null;
  try { body = await res.json(); } catch { body = null; }
  return { status: res.status, body };
}

async function listInstallations(jwt, pages = 10) {
  const out = [];
  for (let page = 1; page <= pages; page += 1) {
    const { status, body } = await get(jwt, `/app/installations?per_page=100&page=${page}`);
    if (status !== 200 || !Array.isArray(body)) {
      if (page === 1) {
        console.error(`GET /app/installations returned ${status}; the JWT is ` +
          'the credential this endpoint wants');
      }
      break;
    }
    out.push(...body);
    if (body.length < 100) break;
  }
  return out;
}

async function main() {
  const jwt = process.env.GITHUB_APP_JWT;
  if (!jwt) {
    console.error('set GITHUB_APP_JWT to the JWT your own signing code ' +
      'produced. suspended_at lives on the installation record, and ' +
      "installation records are read with the App's JWT");
    process.exitCode = 2;
    return;
  }
  const installationId = process.argv[2] ?? process.env.GITHUB_INSTALLATION_ID ?? null;

  const installations = await listInstallations(jwt);
  const stats = summarize(installations);
  console.log(`${stats.total} installation(s) visible to this App, ` +
    `${stats.suspended} suspended`);

  let probeStatus = null;
  const token = process.env.GITHUB_INSTALLATION_TOKEN;
  if (token) {
    ({ status: probeStatus } = await get(token, '/installation/repositories?per_page=1'));
    console.log('installation token: GET /installation/repositories returned ' +
      `${probeStatus}`);
  }

  const now = Date.now();
  const findings = [];
  if (installationId) {
    const target = find(installations, installationId);
    const [state, detail] = verdict(target, probeStatus, now);
    findings.push({ installation_id: installationId, state, detail, retryable: retryable(state) });
    console.log(`${state}: ${detail}`);
    console.log(`repair: ${repair(state, target)}`);
  } else {
    for (const inst of installations) {
      const [state, detail] = verdict(inst, null, now);
      findings.push({ installation_id: inst.id, state, detail, retryable: retryable(state) });
      if (state !== 'active') {
        console.log(`${state}: ${detail}`);
        console.log(`repair: ${repair(state, inst)}`);
      }
    }
    if (stats.suspended === 0) {
      console.log('active: no installation of this App is suspended');
    }
  }

  console.log(JSON.stringify({
    visible: stats.total,
    suspended: stats.suspended,
    suspended_ids: stats.suspended_ids,
    probe_status: probeStatus,
    findings,
  }, null, 2));
  process.exitCode = findings.some((f) => !f.retryable) ? 1 : 0;
}

// Only run when invoked directly, so importing this module from the test file
// does not execute main(), fail on the missing JWT and set an exit code that
// fails the suite even as every test passes.
if (import.meta.url === `file://${process.argv[1]}`) {
  main().catch((err) => { console.error(err.message); process.exitCode = 2; });
}

from datetime import datetime, timezone

from github_installation_suspension import (
    account_of, days_since, find, is_suspended, repair, retryable,
    summarize, suspended_at, suspended_by, verdict,
)

NOW = datetime(2026, 8, 30, 12, 0, 0, tzinfo=timezone.utc)
LIVE = {"id": 41234567, "account": {"login": "acme-corp"}, "suspended_at": None}
DEAD = {"id": 41234568, "account": {"login": "beta-inc"},
        "suspended_at": "2026-08-27T09:14:22Z",
        "suspended_by": {"login": "octo-admin"}}


def test_every_shape_of_absent_timestamp_means_not_suspended():
    assert not is_suspended({"id": 1})
    assert not is_suspended({"id": 1, "suspended_at": None})
    assert not is_suspended({"id": 1, "suspended_at": ""})
    assert not is_suspended({"id": 1, "suspended_at": "   "})
    assert not is_suspended({"id": 1, "suspended_at": "null"})


def test_a_real_timestamp_is_a_suspension():
    assert is_suspended(DEAD)
    assert suspended_at(DEAD) == "2026-08-27T09:14:22Z"


def test_a_suspension_with_no_named_actor_is_still_a_suspension():
    anon = {"id": 9, "suspended_at": "2026-08-27T09:14:22Z", "suspended_by": None}
    assert is_suspended(anon)
    assert suspended_by(anon) is None
    assert suspended_by(DEAD) == "octo-admin"


def test_the_age_is_measured_from_the_timestamp():
    assert days_since("2026-08-27T09:14:22Z", NOW) == 3
    assert days_since("not a date", NOW) is None
    assert days_since(None, NOW) is None


def test_an_id_matches_whether_it_was_stored_as_text_or_a_number():
    assert find([LIVE, DEAD], 41234568) is DEAD
    assert find([LIVE, DEAD], "41234568") is DEAD
    assert find([LIVE, DEAD], " 41234568 ") is DEAD
    assert find([LIVE, DEAD], 999) is None


def test_the_summary_counts_both_sides():
    stats = summarize([LIVE, DEAD, {"id": 3}])
    assert stats == {"total": 3, "suspended": 1, "active": 2,
                     "suspended_ids": [41234568]}


def test_a_suspended_installation_names_the_moment_and_the_actor():
    state, detail = verdict(DEAD, None, NOW)
    assert state == "suspended"
    assert "octo-admin" in detail
    assert "3 day(s) ago" in detail
    assert not retryable(state)


def test_a_missing_id_is_never_reported_as_a_suspension():
    state, detail = verdict(None, 403, NOW)
    assert state == "not-listed"
    assert "different repair" in detail
    assert not retryable(state)


def test_a_403_on_an_active_installation_is_sent_elsewhere():
    state, detail = verdict(LIVE, 403, NOW)
    assert state == "active-but-refused"
    assert "rather than about suspension" in detail
    assert retryable(state)


def test_an_active_installation_with_no_probe_is_just_active():
    assert verdict(LIVE, None, NOW)[0] == "active"
    assert retryable("active")


def test_the_repair_for_a_suspension_names_the_account_and_forbids_retrying():
    text = repair("suspended", DEAD)
    assert "beta-inc" in text
    assert "Retrying cannot help" in text


def test_the_account_falls_back_rather_than_raising():
    assert account_of({"id": 1}) == "an unnamed account"
    assert account_of(None) == "an unnamed account"

import { test } from 'node:test';
import assert from 'node:assert/strict';
import {
  accountOf, daysSince, find, isSuspended, repair, retryable,
  summarize, suspendedAt, suspendedBy, verdict,
} from './github-installation-suspension.mjs';

const NOW = Date.parse('2026-08-30T12:00:00Z');
const LIVE = { id: 41234567, account: { login: 'acme-corp' }, suspended_at: null };
const DEAD = {
  id: 41234568,
  account: { login: 'beta-inc' },
  suspended_at: '2026-08-27T09:14:22Z',
  suspended_by: { login: 'octo-admin' },
};

test('every shape of absent timestamp means not suspended', () => {
  assert.ok(!isSuspended({ id: 1 }));
  assert.ok(!isSuspended({ id: 1, suspended_at: null }));
  assert.ok(!isSuspended({ id: 1, suspended_at: '' }));
  assert.ok(!isSuspended({ id: 1, suspended_at: '   ' }));
  assert.ok(!isSuspended({ id: 1, suspended_at: 'null' }));
});

test('a real timestamp is a suspension', () => {
  assert.ok(isSuspended(DEAD));
  assert.equal(suspendedAt(DEAD), '2026-08-27T09:14:22Z');
});

test('a suspension with no named actor is still a suspension', () => {
  const anon = { id: 9, suspended_at: '2026-08-27T09:14:22Z', suspended_by: null };
  assert.ok(isSuspended(anon));
  assert.equal(suspendedBy(anon), null);
  assert.equal(suspendedBy(DEAD), 'octo-admin');
});

test('the age is measured from the timestamp', () => {
  assert.equal(daysSince('2026-08-27T09:14:22Z', NOW), 3);
  assert.equal(daysSince('not a date', NOW), null);
  assert.equal(daysSince(null, NOW), null);
});

test('an id matches whether it was stored as text or a number', () => {
  assert.equal(find([LIVE, DEAD], 41234568), DEAD);
  assert.equal(find([LIVE, DEAD], '41234568'), DEAD);
  assert.equal(find([LIVE, DEAD], ' 41234568 '), DEAD);
  assert.equal(find([LIVE, DEAD], 999), null);
});

test('the summary counts both sides', () => {
  assert.deepEqual(summarize([LIVE, DEAD, { id: 3 }]), {
    total: 3, suspended: 1, active: 2, suspended_ids: [41234568],
  });
});

test('a suspended installation names the moment and the actor', () => {
  const [state, detail] = verdict(DEAD, null, NOW);
  assert.equal(state, 'suspended');
  assert.match(detail, /octo-admin/);
  assert.match(detail, /3 day\(s\) ago/);
  assert.ok(!retryable(state));
});

test('a missing id is never reported as a suspension', () => {
  const [state, detail] = verdict(null, 403, NOW);
  assert.equal(state, 'not-listed');
  assert.match(detail, /different repair/);
  assert.ok(!retryable(state));
});

test('a 403 on an active installation is sent elsewhere', () => {
  const [state, detail] = verdict(LIVE, 403, NOW);
  assert.equal(state, 'active-but-refused');
  assert.match(detail, /rather than about suspension/);
  assert.ok(retryable(state));
});

test('an active installation with no probe is just active', () => {
  assert.equal(verdict(LIVE, null, NOW)[0], 'active');
  assert.ok(retryable('active'));
});

test('the repair for a suspension names the account and forbids retrying', () => {
  const text = repair('suspended', DEAD);
  assert.match(text, /beta-inc/);
  assert.match(text, /Retrying cannot help/);
});

test('the account falls back rather than throwing', () => {
  assert.equal(accountOf({ id: 1 }), 'an unnamed account');
  assert.equal(accountOf(null), 'an unnamed account');
});