org_login_required: hold the installation, do not retire it

"""Decide what a scheduler should do with a Slack installation that is migrating.

Read only. One GET per installation: auth.test, which is the cheapest call in
the API and the only one worth making against a workspace that has told you it
cannot answer.

The point of the script is one boundary. org_login_required and
team_added_to_org mean the installation is temporarily unreachable; token_revoked
and account_inactive mean the grant has ended. A pipeline that cannot tell those
apart eventually deletes a paying customer in the middle of the migration that
made them an Enterprise account.
"""
import argparse
import json
import logging
import os
import sys

import requests

logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s")
log = logging.getLogger("slack_migration_hold")

API = "https://slack.com/api/"

# The workspace is migrating into an organization. Temporary, and measured in
# hours. Nothing in this set may ever produce a disposition that deletes a row.
HOLD_ERRORS = ("org_login_required", "team_added_to_org")

# The grant has ended. These are the only errors that justify retiring a row.
DEAD_ERRORS = ("token_revoked", "account_inactive")

# Try the same call again shortly; nothing about the installation is wrong.
RETRY_ERRORS = ("ratelimited", "service_unavailable", "internal_error", "fatal_error",
                "request_timeout", "unparseable_body")

# Somebody has to do something, and it is not the scheduler.
ESCALATE_ERRORS = ("missing_scope", "not_allowed_token_type", "enterprise_is_restricted",
                   "app_access_restricted", "ekm_access_denied", "not_an_admin",
                   "feature_not_enabled", "accesslimited")

# Could be a rotation that lapsed, could be a credential that was replaced. A
# human decides; the scheduler does not delete.
INVESTIGATE_ERRORS = ("invalid_auth", "token_expired", "not_authed")

MINUTES_PER_HOUR = 60


def disposition(error):
    """What should the scheduler do with this installation? Pure.

    Returns (action, detail). The whole note lives in the boundary between hold
    and retire: no error in HOLD_ERRORS may ever produce retire.

      serve       nothing is wrong.
      hold        temporarily unreachable; suspend the work.
      retry       transient; try the same call again shortly.
      retire      the grant has ended and the row is dead.
      escalate    a policy or configuration decision somebody has to make.
      investigate ambiguous credential state; a human looks, nothing is deleted.
      unknown     unrecognised; treated as investigate rather than as death.
    """
    name = str(error or "").strip()
    if not name:
        return ("serve", "ok: true, so the installation is answering normally")
    if name in HOLD_ERRORS:
        return ("hold", "%s: the workspace is migrating into an organization and will "
                        "answer once it settles; suspend the work, do not fail it"
                % name)
    if name in DEAD_ERRORS:
        return ("retire", "%s: the grant has ended and the row is dead" % name)
    if name in RETRY_ERRORS:
        return ("retry", "%s: transient, so the same call is worth making again "
                         "shortly" % name)
    if name in ESCALATE_ERRORS:
        return ("escalate", "%s: a policy or configuration decision that no amount of "
                            "retrying will change" % name)
    if name in INVESTIGATE_ERRORS:
        return ("investigate", "%s: the credential state is ambiguous; a person looks "
                               "at it and nothing is deleted on the strength of it"
                % name)
    return ("unknown", "%s is not in this map; treated as investigate, because the "
                       "safe default for an unrecognised failure is to keep the row"
            % name)


def backoff_seconds(attempt, base=900, cap=14400):
    """How long to wait before the next probe. Pure.

    Fifteen minutes, doubling, capped at four hours. The cap is the point: an
    uncapped exponential eventually stops checking, and a five second retry is
    an attack on a workspace that is already having a difficult day.
    """
    try:
        n = int(attempt)
    except (TypeError, ValueError):
        n = 0
    n = max(0, min(n, 30))
    return int(min(cap, base * (2 ** n)))


def hold_state(observations, give_up_hours=72):
    """Is this installation held now, and for how long? Pure.

    observations: [(minute, error), ...] where an empty error means the call
    succeeded. Returns (state, minutes).

      not-held        no hold error was ever recorded.
      held            the most recent observation is a hold error.
      long-hold       held for longer than give_up_hours. A person should look;
                      the row is still not deleted.
      cleared         a hold was recorded and the latest observation succeeded.
      nothing-observed  the log is empty.
    """
    seen = sorted(((int(m), str(e or "")) for m, e in (observations or [])),
                  key=lambda row: row[0])
    if not seen:
        return ("nothing-observed", 0)
    holds = [row for row in seen if row[1] in HOLD_ERRORS]
    if not holds:
        return ("not-held", 0)
    span = seen[-1][0] - holds[0][0]
    if seen[-1][1] not in HOLD_ERRORS:
        return ("cleared", span)
    if span >= give_up_hours * MINUTES_PER_HOUR:
        return ("long-hold", span)
    return ("held", span)


def resume_actions(before, after):
    """What has to happen before the suspended work restarts? Pure.

    `before` is the identity you stored, `after` is a fresh auth.test body.
    Returns (state, actions).
    """
    was = before or {}
    now = after or {}
    if now.get("ok") is not True:
        return ("not-resumable", ["the installation is still not answering; keep "
                                  "holding rather than resuming"])
    was_ent = str(was.get("enterprise_id") or "")
    now_ent = str(now.get("enterprise_id") or "")
    actions, state = [], "identity-unchanged"
    if now_ent and not was_ent:
        state = "enterprise-id-appeared"
        actions.append("the workspace now belongs to organization %s and the stored "
                       "row has no enterprise_id; re-key the installation before "
                       "resuming" % now_ent)
    elif now_ent != was_ent:
        state = "enterprise-id-changed"
        actions.append("stored enterprise_id is %r and the token reports %r; the row "
                       "does not describe this installation"
                       % (was_ent or None, now_ent or None))
    if str(was.get("team_id") or "") != str(now.get("team_id") or ""):
        if state == "identity-unchanged":
            state = "team-id-changed"
        actions.append("the workspace id moved from %s to %s; every cached channel id "
                       "for it is suspect"
                       % (was.get("team_id") or "<none>", now.get("team_id") or
                          "<none>"))
    if str(was.get("user_id") or "") != str(now.get("user_id") or ""):
        if state == "identity-unchanged":
            state = "user-id-changed"
        actions.append("the authorising user id moved from %s to %s; cached user ids "
                       "need re-resolving, which is a job of its own"
                       % (was.get("user_id") or "<none>", now.get("user_id") or
                          "<none>"))
    if not actions:
        actions.append("the identity is unchanged; resume the suspended work as it "
                       "was left")
    return (state, actions)


def queue_verdict(rows):
    """One sentence about the whole installation store. Pure.

    rows: [(install, action), ...]. Returns (state, counts).
    """
    seen = [(str(i), str(a)) for i, a in (rows or [])]
    counts = {"installs": len(seen), "held": 0, "serving": 0, "retiring": 0,
              "other": 0}
    for _install, action in seen:
        if action == "hold":
            counts["held"] += 1
        elif action == "serve":
            counts["serving"] += 1
        elif action == "retire":
            counts["retiring"] += 1
        else:
            counts["other"] += 1
    if not seen:
        return ("empty", counts)
    if not counts["held"]:
        return ("serving", counts)
    if counts["held"] == len(seen):
        return ("all-held", counts)
    return ("some-held", counts)


def get(session, method, token, params=None):
    """One GET against the Web API. Returns the parsed body."""
    r = session.get(API + method, params=params or {},
                    headers={"Authorization": "Bearer " + token}, timeout=30)
    try:
        return r.json()
    except ValueError:
        return {"ok": False, "error": "unparseable_body"}


def main():
    ap = argparse.ArgumentParser(description=__doc__)
    ap.add_argument("--store", help="JSON array of installation rows; each row needs "
                                    "an install name and a token_env")
    ap.add_argument("--log", help="JSON object of install -> [[minute, error], ...] "
                                  "observations you have already recorded")
    ap.add_argument("--token-env", default="SLACK_BOT_TOKEN",
                    help="used when --store is not given")
    args = ap.parse_args()

    rows = []
    if args.store:
        with open(args.store, encoding="utf-8") as fh:
            rows = json.load(fh)
    else:
        rows = [{"install": "<the only row>", "token_env": args.token_env}]

    observations = {}
    if args.log:
        with open(args.log, encoding="utf-8") as fh:
            observations = json.load(fh)

    session = requests.Session()
    verdicts, findings = [], []
    for row in rows:
        install = str(row.get("install") or row.get("team_id") or "<unnamed>")
        token = os.environ.get(row.get("token_env") or args.token_env)
        if not token:
            log.warning("install    %-18s no-token    %s is unset", install,
                        row.get("token_env") or args.token_env)
            verdicts.append((install, "investigate"))
            continue
        body = get(session, "auth.test", token)
        error = "" if body.get("ok") is True else str(body.get("error") or "unknown")
        action, why = disposition(error)
        verdicts.append((install, action))
        (log.info if action == "serve" else log.warning)(
            "install    %-18s %-11s %s", install, action, why)

        if action == "hold":
            state, minutes = hold_state(observations.get(install) or
                                        [[0, error]])
            log.warning("hold       %-18s %-11s %d minute(s) since the first hold was "
                        "recorded", install, state, minutes)
            attempt = len(observations.get(install) or [])
            log.warning("backoff    %-18s %-11s capped; one auth.test per interval is "
                        "enough to notice the moment it clears", install,
                        "%ds" % backoff_seconds(attempt))
        elif action == "serve" and observations.get(install):
            state, minutes = hold_state(observations[install] + [[10 ** 6, ""]])
            if state == "cleared":
                resume, actions = resume_actions(row, body)
                log.warning("resume     %-18s %-11s after %d minute(s) held", install,
                            resume, minutes)
                for line in actions:
                    log.warning("           %s", line)
                    if resume != "identity-unchanged":
                        findings.append(line)

    summary, counts = queue_verdict(verdicts)
    (log.info if summary == "serving" else log.warning)(
        "queue      %-18s %d held, %d serving, %d retiring, of %d", summary,
        counts["held"], counts["serving"], counts["retiring"], counts["installs"])

    if counts["held"]:
        findings.append("suspend scheduled work for %d held installation(s); a held "
                        "row must never be eligible for the cleanup that removes dead "
                        "ones" % counts["held"])
        findings.append("when the hold clears, re-run auth.test and compare the "
                        "identity before resuming; a workspace that has just joined "
                        "an organization has an enterprise_id it did not have")
    if not findings:
        log.info("verdict    clean              nothing is held and nothing needs "
                 "re-keying")
        return 0
    log.warning("verdict    %d finding(s)", len(findings))
    for line in findings:
        log.warning("  repair: %s", line)
    return 1


if __name__ == "__main__":
    sys.exit(main())

/**
 * Decide what a scheduler should do with a Slack installation that is migrating.
 *
 * Read only. One GET per installation: auth.test, which is the cheapest call in
 * the API and the only one worth making against a workspace that has told you
 * it cannot answer.
 *
 * The point of the script is one boundary. org_login_required and
 * team_added_to_org mean the installation is temporarily unreachable;
 * token_revoked and account_inactive mean the grant has ended.
 */

import { readFile } from 'node:fs/promises';

const API = 'https://slack.com/api/';

// Temporary, measured in hours. Nothing here may ever produce a disposition
// that deletes a row.
export const HOLD_ERRORS = ['org_login_required', 'team_added_to_org'];

// The grant has ended. The only errors that justify retiring a row.
export const DEAD_ERRORS = ['token_revoked', 'account_inactive'];

export const RETRY_ERRORS = ['ratelimited', 'service_unavailable', 'internal_error',
  'fatal_error', 'request_timeout', 'unparseable_body'];

export const ESCALATE_ERRORS = ['missing_scope', 'not_allowed_token_type',
  'enterprise_is_restricted', 'app_access_restricted', 'ekm_access_denied',
  'not_an_admin', 'feature_not_enabled', 'accesslimited'];

export const INVESTIGATE_ERRORS = ['invalid_auth', 'token_expired', 'not_authed'];

const MINUTES_PER_HOUR = 60;

/**
 * What should the scheduler do with this installation? Pure.
 * Returns [action, detail]; serve, hold, retry, retire, escalate, investigate,
 * unknown.
 */
export function disposition(error) {
  const name = String(error ?? '').trim();
  if (!name) return ['serve', 'ok: true, so the installation is answering normally'];
  if (HOLD_ERRORS.includes(name)) {
    return ['hold', `${name}: the workspace is migrating into an organization and `
      + 'will answer once it settles; suspend the work, do not fail it'];
  }
  if (DEAD_ERRORS.includes(name)) {
    return ['retire', `${name}: the grant has ended and the row is dead`];
  }
  if (RETRY_ERRORS.includes(name)) {
    return ['retry', `${name}: transient, so the same call is worth making again `
      + 'shortly'];
  }
  if (ESCALATE_ERRORS.includes(name)) {
    return ['escalate', `${name}: a policy or configuration decision that no amount `
      + 'of retrying will change'];
  }
  if (INVESTIGATE_ERRORS.includes(name)) {
    return ['investigate', `${name}: the credential state is ambiguous; a person `
      + 'looks at it and nothing is deleted on the strength of it'];
  }
  return ['unknown', `${name} is not in this map; treated as investigate, because the `
    + 'safe default for an unrecognised failure is to keep the row'];
}

/**
 * How long to wait before the next probe. Pure.
 * Fifteen minutes, doubling, capped at four hours.
 */
export function backoffSeconds(attempt, base = 900, cap = 14400) {
  let n = Number.parseInt(attempt, 10);
  if (!Number.isFinite(n)) n = 0;
  n = Math.max(0, Math.min(n, 30));
  return Math.min(cap, base * (2 ** n));
}

/**
 * Is this installation held now, and for how long? Pure.
 * observations: [[minute, error], ...]. Returns [state, minutes].
 */
export function holdState(observations, giveUpHours = 72) {
  const seen = (observations ?? [])
    .map(([m, e]) => [Number.parseInt(m, 10) || 0, String(e ?? '')])
    .sort((a, b) => a[0] - b[0]);
  if (!seen.length) return ['nothing-observed', 0];
  const holds = seen.filter(([, e]) => HOLD_ERRORS.includes(e));
  if (!holds.length) return ['not-held', 0];
  const span = seen[seen.length - 1][0] - holds[0][0];
  if (!HOLD_ERRORS.includes(seen[seen.length - 1][1])) return ['cleared', span];
  if (span >= giveUpHours * MINUTES_PER_HOUR) return ['long-hold', span];
  return ['held', span];
}

/**
 * What has to happen before the suspended work restarts? Pure.
 * Returns [state, actions].
 */
export function resumeActions(before, after) {
  const was = before ?? {};
  const now = after ?? {};
  if (now.ok !== true) {
    return ['not-resumable', ['the installation is still not answering; keep holding '
      + 'rather than resuming']];
  }
  const wasEnt = String(was.enterprise_id ?? '');
  const nowEnt = String(now.enterprise_id ?? '');
  const actions = [];
  let state = 'identity-unchanged';
  if (nowEnt && !wasEnt) {
    state = 'enterprise-id-appeared';
    actions.push(`the workspace now belongs to organization ${nowEnt} and the stored `
      + 'row has no enterprise_id; re-key the installation before resuming');
  } else if (nowEnt !== wasEnt) {
    state = 'enterprise-id-changed';
    actions.push(`stored enterprise_id is ${wasEnt || 'None'} and the token reports `
      + `${nowEnt || 'None'}; the row does not describe this installation`);
  }
  if (String(was.team_id ?? '') !== String(now.team_id ?? '')) {
    if (state === 'identity-unchanged') state = 'team-id-changed';
    actions.push(`the workspace id moved from ${was.team_id ?? '<none>'} to `
      + `${now.team_id ?? '<none>'}; every cached channel id for it is suspect`);
  }
  if (String(was.user_id ?? '') !== String(now.user_id ?? '')) {
    if (state === 'identity-unchanged') state = 'user-id-changed';
    actions.push(`the authorising user id moved from ${was.user_id ?? '<none>'} to `
      + `${now.user_id ?? '<none>'}; cached user ids need re-resolving, which is a `
      + 'job of its own');
  }
  if (!actions.length) {
    actions.push('the identity is unchanged; resume the suspended work as it was left');
  }
  return [state, actions];
}

/**
 * One sentence about the whole installation store. Pure.
 * Returns [state, counts].
 */
export function queueVerdict(rows) {
  const seen = (rows ?? []).map(([i, a]) => [String(i), String(a)]);
  const counts = {
    installs: seen.length, held: 0, serving: 0, retiring: 0, other: 0,
  };
  for (const [, action] of seen) {
    if (action === 'hold') counts.held += 1;
    else if (action === 'serve') counts.serving += 1;
    else if (action === 'retire') counts.retiring += 1;
    else counts.other += 1;
  }
  if (!seen.length) return ['empty', counts];
  if (!counts.held) return ['serving', counts];
  if (counts.held === seen.length) return ['all-held', counts];
  return ['some-held', counts];
}

function arg(args, name, fallback = '') {
  const i = args.indexOf(name);
  return i === -1 ? fallback : args[i + 1];
}

async function read(token, method, params) {
  const qs = new URLSearchParams(params ?? {}).toString();
  const r = await fetch(`${API}${method}${qs ? `?${qs}` : ''}`,
    { headers: { Authorization: `Bearer ${token}` } });
  try {
    return await r.json();
  } catch {
    return { ok: false, error: 'unparseable_body' };
  }
}

async function main() {
  const args = process.argv.slice(2);
  const defaultEnv = arg(args, '--token-env', 'SLACK_BOT_TOKEN');
  const storePath = arg(args, '--store');
  const logPath = arg(args, '--log');

  const rows = storePath
    ? JSON.parse(await readFile(storePath, 'utf8'))
    : [{ install: '<the only row>', token_env: defaultEnv }];
  const observations = logPath ? JSON.parse(await readFile(logPath, 'utf8')) : {};

  const verdicts = [];
  const findings = [];
  for (const row of rows) {
    const install = String(row.install ?? row.team_id ?? '<unnamed>');
    const token = process.env[row.token_env ?? defaultEnv];
    if (!token) {
      console.warn(`install    ${install.padEnd(18)} no-token    `
        + `${row.token_env ?? defaultEnv} is unset`);
      verdicts.push([install, 'investigate']);
      continue;
    }
    // eslint-disable-next-line no-await-in-loop
    const body = await read(token, 'auth.test', {});
    const error = body.ok === true ? '' : String(body.error ?? 'unknown');
    const [action, why] = disposition(error);
    verdicts.push([install, action]);
    const line = `install    ${install.padEnd(18)} ${action.padEnd(11)} ${why}`;
    if (action === 'serve') console.log(line);
    else console.warn(line);

    if (action === 'hold') {
      const [state, minutes] = holdState(observations[install] ?? [[0, error]]);
      console.warn(`hold       ${install.padEnd(18)} ${state.padEnd(11)} ${minutes} `
        + 'minute(s) since the first hold was recorded');
      const attempt = (observations[install] ?? []).length;
      console.warn(`backoff    ${install.padEnd(18)} `
        + `${`${backoffSeconds(attempt)}s`.padEnd(11)} capped; one auth.test per `
        + 'interval is enough to notice the moment it clears');
    } else if (action === 'serve' && observations[install]) {
      const [state, minutes] = holdState([...observations[install], [10 ** 6, '']]);
      if (state === 'cleared') {
        const [resume, actions] = resumeActions(row, body);
        console.warn(`resume     ${install.padEnd(18)} ${resume.padEnd(11)} after `
          + `${minutes} minute(s) held`);
        for (const action2 of actions) {
          console.warn(`           ${action2}`);
          if (resume !== 'identity-unchanged') findings.push(action2);
        }
      }
    }
  }

  const [summary, counts] = queueVerdict(verdicts);
  const summaryLine = `queue      ${summary.padEnd(18)} ${counts.held} held, `
    + `${counts.serving} serving, ${counts.retiring} retiring, of ${counts.installs}`;
  if (summary === 'serving') console.log(summaryLine);
  else console.warn(summaryLine);

  if (counts.held) {
    findings.push(`suspend scheduled work for ${counts.held} held installation(s); a `
      + 'held row must never be eligible for the cleanup that removes dead ones');
    findings.push('when the hold clears, re-run auth.test and compare the identity '
      + 'before resuming; a workspace that has just joined an organization has an '
      + 'enterprise_id it did not have');
  }
  if (!findings.length) {
    console.log('verdict    clean              nothing is held and nothing needs '
      + 're-keying');
    return;
  }
  console.warn(`verdict    ${findings.length} finding(s)`);
  for (const line of findings) console.warn(`  repair: ${line}`);
  process.exitCode = 1;
}

// Only run when invoked directly, so importing this module in the tests does
// not execute main().
if (import.meta.url === `file://${process.argv[1]}`) {
  main().catch((err) => { console.error(err.message); process.exitCode = 2; });
}

from slack_migration_hold import (
    DEAD_ERRORS, HOLD_ERRORS, backoff_seconds, disposition, hold_state, queue_verdict,
    resume_actions,
)


def test_a_successful_call_is_served():
    state, detail = disposition("")
    assert state == "serve"
    assert "answering normally" in detail


def test_the_migration_errors_are_held_and_say_suspend_not_fail():
    for error in HOLD_ERRORS:
        state, detail = disposition(error)
        assert state == "hold"
        assert "do not fail it" in detail


def test_no_hold_error_can_ever_delete_a_row():
    for error in HOLD_ERRORS:
        assert disposition(error)[0] not in ("retire",)


def test_only_the_dead_errors_retire_a_row():
    for error in DEAD_ERRORS:
        assert disposition(error)[0] == "retire"


def test_a_rate_limit_is_a_retry_rather_than_a_hold():
    assert disposition("ratelimited")[0] == "retry"


def test_a_policy_refusal_escalates_instead_of_retrying():
    state, detail = disposition("app_access_restricted")
    assert state == "escalate"
    assert "no amount of retrying" in detail


def test_an_ambiguous_credential_is_investigated_not_deleted():
    for error in ("invalid_auth", "token_expired"):
        state, detail = disposition(error)
        assert state == "investigate"
        assert "nothing is deleted" in detail


def test_an_unrecognised_error_keeps_the_row():
    state, detail = disposition("some_new_error")
    assert state == "unknown"
    assert "keep the row" in detail


def test_the_first_backoff_is_a_quarter_of_an_hour():
    assert backoff_seconds(0) == 900


def test_the_backoff_doubles():
    assert backoff_seconds(1) == 1800
    assert backoff_seconds(2) == 3600


def test_the_backoff_is_capped_in_hours():
    assert backoff_seconds(20) == 14400
    assert backoff_seconds(400) == 14400


def test_a_nonsense_attempt_count_does_not_produce_a_tight_loop():
    assert backoff_seconds(-9) == 900
    assert backoff_seconds(None) == 900


def test_an_installation_with_no_hold_recorded_is_not_held():
    assert hold_state([[0, ""], [30, "ratelimited"]])[0] == "not-held"


def test_a_current_hold_carries_its_span():
    state, minutes = hold_state([[0, "org_login_required"],
                                 [1140, "org_login_required"]])
    assert state == "held"
    assert minutes == 1140


def test_a_hold_that_has_cleared_is_reported_as_cleared():
    state, minutes = hold_state([[0, "team_added_to_org"], [600, ""]])
    assert state == "cleared"
    assert minutes == 600


def test_a_hold_past_three_days_escalates_and_still_does_not_delete():
    state, _ = hold_state([[0, "org_login_required"], [72 * 60, "org_login_required"]])
    assert state == "long-hold"


def test_the_give_up_threshold_is_a_parameter_not_a_constant():
    assert hold_state([[0, "org_login_required"], [120, "org_login_required"]],
                      give_up_hours=1)[0] == "long-hold"


def test_observations_are_sorted_before_they_are_read():
    state, minutes = hold_state([[600, ""], [0, "org_login_required"]])
    assert state == "cleared"
    assert minutes == 600


def test_an_empty_log_concludes_nothing():
    assert hold_state([])[0] == "nothing-observed"


def test_an_enterprise_id_appearing_is_the_signature_of_the_migration():
    state, actions = resume_actions({"team_id": "T04MKTG"},
                                    {"ok": True, "team_id": "T04MKTG",
                                     "enterprise_id": "E04NORTHWIND"})
    assert state == "enterprise-id-appeared"
    assert any("re-key" in line for line in actions)


def test_a_changed_workspace_id_makes_cached_channels_suspect():
    state, actions = resume_actions({"team_id": "T04MKTG"},
                                    {"ok": True, "team_id": "T04NEW"})
    assert state == "team-id-changed"
    assert any("cached channel id" in line for line in actions)


def test_a_changed_user_id_is_handed_to_a_job_of_its_own():
    state, actions = resume_actions({"team_id": "T04MKTG", "user_id": "U01"},
                                    {"ok": True, "team_id": "T04MKTG",
                                     "user_id": "W01"})
    assert state == "user-id-changed"
    assert any("job of its own" in line for line in actions)


def test_an_unchanged_identity_simply_resumes():
    state, actions = resume_actions({"team_id": "T04MKTG"},
                                    {"ok": True, "team_id": "T04MKTG"})
    assert state == "identity-unchanged"
    assert "resume the suspended work" in actions[0]


def test_an_installation_that_still_will_not_answer_is_not_resumable():
    state, actions = resume_actions({"team_id": "T04MKTG"},
                                    {"ok": False, "error": "org_login_required"})
    assert state == "not-resumable"
    assert "keep holding" in actions[0]


def test_a_store_with_nothing_held_is_serving():
    state, counts = queue_verdict([("T04MKTG", "serve"), ("T09OPS", "serve")])
    assert state == "serving"
    assert counts["serving"] == 2


def test_a_store_with_one_hold_is_reported_as_such():
    state, counts = queue_verdict([("T04MKTG", "hold"), ("T09OPS", "serve"),
                                   ("T02OLD", "retire")])
    assert state == "some-held"
    assert counts["held"] == 1
    assert counts["retiring"] == 1


def test_every_installation_held_at_once_is_its_own_state():
    assert queue_verdict([("T04MKTG", "hold"), ("T09OPS", "hold")])[0] == "all-held"


def test_an_empty_store_says_so():
    assert queue_verdict([])[0] == "empty"

import { test } from 'node:test';
import assert from 'node:assert/strict';
import {
  DEAD_ERRORS, HOLD_ERRORS, backoffSeconds, disposition, holdState, queueVerdict,
  resumeActions,
} from './slack-migration-hold.mjs';

test('a successful call is served', () => {
  const [state, detail] = disposition('');
  assert.equal(state, 'serve');
  assert.match(detail, /answering normally/);
});

test('the migration errors are held and say suspend not fail', () => {
  for (const error of HOLD_ERRORS) {
    const [state, detail] = disposition(error);
    assert.equal(state, 'hold');
    assert.match(detail, /do not fail it/);
  }
});

test('no hold error can ever delete a row', () => {
  for (const error of HOLD_ERRORS) {
    assert.notEqual(disposition(error)[0], 'retire');
  }
});

test('only the dead errors retire a row', () => {
  for (const error of DEAD_ERRORS) {
    assert.equal(disposition(error)[0], 'retire');
  }
});

test('a rate limit is a retry rather than a hold', () => {
  assert.equal(disposition('ratelimited')[0], 'retry');
});

test('a policy refusal escalates instead of retrying', () => {
  const [state, detail] = disposition('app_access_restricted');
  assert.equal(state, 'escalate');
  assert.match(detail, /no amount of retrying/);
});

test('an ambiguous credential is investigated not deleted', () => {
  for (const error of ['invalid_auth', 'token_expired']) {
    const [state, detail] = disposition(error);
    assert.equal(state, 'investigate');
    assert.match(detail, /nothing is deleted/);
  }
});

test('an unrecognised error keeps the row', () => {
  const [state, detail] = disposition('some_new_error');
  assert.equal(state, 'unknown');
  assert.match(detail, /keep the row/);
});

test('the first backoff is a quarter of an hour', () => {
  assert.equal(backoffSeconds(0), 900);
});

test('the backoff doubles', () => {
  assert.equal(backoffSeconds(1), 1800);
  assert.equal(backoffSeconds(2), 3600);
});

test('the backoff is capped in hours', () => {
  assert.equal(backoffSeconds(20), 14400);
  assert.equal(backoffSeconds(400), 14400);
});

test('a nonsense attempt count does not produce a tight loop', () => {
  assert.equal(backoffSeconds(-9), 900);
  assert.equal(backoffSeconds(null), 900);
});

test('an installation with no hold recorded is not held', () => {
  assert.equal(holdState([[0, ''], [30, 'ratelimited']])[0], 'not-held');
});

test('a current hold carries its span', () => {
  const [state, minutes] = holdState([[0, 'org_login_required'],
    [1140, 'org_login_required']]);
  assert.equal(state, 'held');
  assert.equal(minutes, 1140);
});

test('a hold that has cleared is reported as cleared', () => {
  const [state, minutes] = holdState([[0, 'team_added_to_org'], [600, '']]);
  assert.equal(state, 'cleared');
  assert.equal(minutes, 600);
});

test('a hold past three days escalates and still does not delete', () => {
  const [state] = holdState([[0, 'org_login_required'],
    [72 * 60, 'org_login_required']]);
  assert.equal(state, 'long-hold');
});

test('the give up threshold is a parameter not a constant', () => {
  assert.equal(holdState([[0, 'org_login_required'], [120, 'org_login_required']],
    1)[0], 'long-hold');
});

test('observations are sorted before they are read', () => {
  const [state, minutes] = holdState([[600, ''], [0, 'org_login_required']]);
  assert.equal(state, 'cleared');
  assert.equal(minutes, 600);
});

test('an empty log concludes nothing', () => {
  assert.equal(holdState([])[0], 'nothing-observed');
});

test('an enterprise id appearing is the signature of the migration', () => {
  const [state, actions] = resumeActions({ team_id: 'T04MKTG' },
    { ok: true, team_id: 'T04MKTG', enterprise_id: 'E04NORTHWIND' });
  assert.equal(state, 'enterprise-id-appeared');
  assert.equal(actions.some((a) => a.includes('re-key')), true);
});

test('a changed workspace id makes cached channels suspect', () => {
  const [state, actions] = resumeActions({ team_id: 'T04MKTG' },
    { ok: true, team_id: 'T04NEW' });
  assert.equal(state, 'team-id-changed');
  assert.equal(actions.some((a) => a.includes('cached channel id')), true);
});

test('a changed user id is handed to a job of its own', () => {
  const [state, actions] = resumeActions({ team_id: 'T04MKTG', user_id: 'U01' },
    { ok: true, team_id: 'T04MKTG', user_id: 'W01' });
  assert.equal(state, 'user-id-changed');
  assert.equal(actions.some((a) => a.includes('job of its own')), true);
});

test('an unchanged identity simply resumes', () => {
  const [state, actions] = resumeActions({ team_id: 'T04MKTG' },
    { ok: true, team_id: 'T04MKTG' });
  assert.equal(state, 'identity-unchanged');
  assert.match(actions[0], /resume the suspended work/);
});

test('an installation that still will not answer is not resumable', () => {
  const [state, actions] = resumeActions({ team_id: 'T04MKTG' },
    { ok: false, error: 'org_login_required' });
  assert.equal(state, 'not-resumable');
  assert.match(actions[0], /keep holding/);
});

test('a store with nothing held is serving', () => {
  const [state, counts] = queueVerdict([['T04MKTG', 'serve'], ['T09OPS', 'serve']]);
  assert.equal(state, 'serving');
  assert.equal(counts.serving, 2);
});

test('a store with one hold is reported as such', () => {
  const [state, counts] = queueVerdict([['T04MKTG', 'hold'], ['T09OPS', 'serve'],
    ['T02OLD', 'retire']]);
  assert.equal(state, 'some-held');
  assert.equal(counts.held, 1);
  assert.equal(counts.retiring, 1);
});

test('every installation held at once is its own state', () => {
  assert.equal(queueVerdict([['T04MKTG', 'hold'], ['T09OPS', 'hold']])[0], 'all-held');
});

test('an empty store says so', () => {
  assert.equal(queueVerdict([])[0], 'empty');
});