11200 on the TwiML fetch: the call fails, not just a receipt

"""Report 11200 retrieval failures on the TwiML handlers, not the receipts.

Read only. GET requests and nothing else: give this an API Key with read access
rather than the account auth token. The repair is printed, never performed,
because this script holds a credential to an account that can send messages and
spend money.
"""
import argparse
import datetime as dt
import logging
import os
import sys

import requests

logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s")
log = logging.getLogger("twilio_twiml_retrieval_audit")

HOST = "https://api.twilio.com"
BASE = HOST + "/2010-04-01"
MSG = "https://messaging.twilio.com/v1"
MONITOR = "https://monitor.twilio.com/v1"

PRIMARY = {"voice", "sms", "inbound"}


def code_of(alert):
    """error_code arrives as a string on some alerts and an int on others."""
    try:
        return int(alert.get("error_code"))
    except (TypeError, ValueError):
        return None


def endpoint(url):
    """Reduce a URL to lowercase host plus path.

    Pure. Twilio appends nothing, but applications routinely carry a per-call
    query string, and grouping on the raw URL turns one broken handler into
    forty endpoints that each look survivable.
    """
    u = str(url or "").strip()
    for scheme in ("https://", "http://"):
        if u.lower().startswith(scheme):
            u = u[len(scheme):]
            break
    u = u.split("?", 1)[0].split("#", 1)[0]
    if "@" in u.split("/", 1)[0]:
        u = u.split("@", 1)[1]
    return u.rstrip("/").lower()


def handler_index(numbers, services):
    """Map every configured URL to the roles it plays and what it protects.

    Pure. The alert says which URL failed; only this index says whether that URL
    is a TwiML handler, a fallback or a delivery receipt, and whether the thing
    it serves has a fallback behind it.
    """
    idx = {}

    def add(url, role, exposed=None):
        e = endpoint(url)
        if not e:
            return
        entry = idx.setdefault(e, {"roles": set(), "exposed": []})
        entry["roles"].add(role)
        if exposed:
            entry["exposed"].append(exposed)

    for n in numbers:
        label = n.get("phone_number") or n.get("sid") or "?"
        voice_fb = str(n.get("voice_fallback_url") or "").strip()
        sms_fb = str(n.get("sms_fallback_url") or "").strip()
        add(n.get("voice_url"), "voice", None if voice_fb else label + " voice")
        add(n.get("sms_url"), "sms", None if sms_fb else label + " sms")
        add(voice_fb, "fallback")
        add(sms_fb, "fallback")
        add(n.get("status_callback"), "status-callback")
        add(n.get("sms_status_callback"), "status-callback")

    for s in services:
        label = s.get("friendly_name") or s.get("sid") or "?"
        fb = str(s.get("fallback_url") or "").strip()
        add(s.get("inbound_request_url"), "inbound", None if fb else label + " inbound")
        add(fb, "fallback")
        add(s.get("status_callback"), "status-callback")

    return idx


def verdict(row, min_alerts=3):
    """Classify one failing endpoint. Pure, so the severity rule can be tested
    without a network.

    `row` carries the normalised endpoint, the alert count, the roles it plays
    and the handlers it serves that have no fallback. Returns (state, detail).
    """
    roles = set(row.get("roles") or ())
    exposed = list(row.get("exposed") or ())
    n = int(row.get("count") or 0)

    if roles and roles <= {"status-callback"}:
        return ("status-callback",
                "%d failure(s) on a delivery receipt URL. That loses the receipt, "
                "not the call, and it is a different note with a different "
                "repair." % n)

    if not roles:
        return ("unattributed",
                "%d failure(s) on a URL that no number and no Messaging Service "
                "currently points at: a TwiML App, a Studio flow, or a handler "
                "that has since been reconfigured." % n)

    primary = roles & PRIMARY
    if not primary:
        return ("fallback-failing",
                "%d failure(s) on a fallback URL. The fallback is the last thing "
                "between a broken handler and a dropped call, and it is the "
                "thing returning non-2xx." % n)

    where = "/".join(sorted(primary))
    if exposed:
        return ("no-safety-net",
                "%d failure(s) on the %s handler for %s, which has no fallback "
                "URL. Twilio has nothing to execute, so it plays its own error "
                "message and hangs up, or drops the inbound message."
                % (n, where, ", ".join(exposed[:3])))

    if n < min_alerts:
        return ("intermittent",
                "%d failure(s) on the %s handler, and a fallback answered. Under "
                "the %d threshold: noise, until the rate changes."
                % (n, where, min_alerts))

    return ("degraded",
            "%d failure(s) on the %s handler. A fallback answered, so callers "
            "were served something, but not your application." % (n, where))


def get(session, url, **params):
    r = session.get(url, params=params, timeout=30)
    if r.status_code in (401, 403):
        raise SystemExit("%d from Twilio: check TWILIO_ACCOUNT_SID and that the "
                         "API key belongs to that account with read access"
                         % r.status_code)
    r.raise_for_status()
    return r.json()


def list_v1(session, url, key, limit=10000, **params):
    """Page a v1 list. meta.next_page_url is absolute."""
    out = []
    params.setdefault("PageSize", 100)
    while url and len(out) < limit:
        page = get(session, url, **params)
        out.extend(page.get(key, []))
        url, params = (page.get("meta") or {}).get("next_page_url"), {}
    return out[:limit]


def list_numbers(session, account, limit=2000):
    url = "%s/Accounts/%s/IncomingPhoneNumbers.json" % (BASE, account)
    params = {"PageSize": 100}
    out = []
    while url and len(out) < limit:
        page = get(session, url, **params)
        out.extend(page.get("incoming_phone_numbers", []))
        nxt = page.get("next_page_uri")
        url, params = (HOST + nxt) if nxt else None, {}
    return out[:limit]


def fetch_alert(session, sid):
    """response_body and response_headers exist only on the single-alert fetch."""
    return get(session, "%s/Alerts/%s" % (MONITOR, sid))


def main():
    ap = argparse.ArgumentParser(description=__doc__)
    ap.add_argument("--days", type=int, default=3, help="alert window, max 30")
    ap.add_argument("--min-alerts", type=int, default=3,
                    help="below this a fallback-covered endpoint is noise")
    ap.add_argument("--sample", action="store_true",
                    help="one extra GET per endpoint to see the response Twilio got")
    args = ap.parse_args()

    account = os.environ.get("TWILIO_ACCOUNT_SID")
    key = os.environ.get("TWILIO_API_KEY")
    secret = os.environ.get("TWILIO_API_SECRET")
    if not (account and key and secret):
        log.error("set TWILIO_ACCOUNT_SID, TWILIO_API_KEY and TWILIO_API_SECRET "
                  "(an API Key with read access, not the auth token)")
        return 2

    session = requests.Session()
    session.auth = (key, secret)

    since = (dt.datetime.now(dt.timezone.utc)
             - dt.timedelta(days=min(args.days, 30))).strftime("%Y-%m-%dT%H:%M:%SZ")
    alerts = [a for a in list_v1(session, MONITOR + "/Alerts", "alerts",
                                 LogLevel="error", StartDate=since)
              if code_of(a) == 11200]
    if not alerts:
        log.info("no 11200 alerts in the last %d day(s)", args.days)
        return 0

    idx = handler_index(list_numbers(session, account),
                        list_v1(session, MSG + "/Services", "services", 1000))

    rows = {}
    for a in alerts:
        e = endpoint(a.get("request_url"))
        row = rows.setdefault(e, {"endpoint": e, "count": 0, "sid": a.get("sid"),
                                  "roles": set(), "exposed": []})
        row["count"] += 1
        known = idx.get(e)
        if known:
            row["roles"] = known["roles"]
            row["exposed"] = known["exposed"]

    bad = 0
    for row in sorted(rows.values(), key=lambda r: -r["count"]):
        state, detail = verdict(row, args.min_alerts)
        line = "%-16s %s  %s" % (state, row["endpoint"], detail)
        if state in ("intermittent", "status-callback"):
            log.info(line)
            continue
        bad += 1
        log.warning(line)
        if args.sample and row["sid"]:
            full = fetch_alert(session, row["sid"])
            log.warning("  %s returned: %s", full.get("request_method") or "GET",
                        str(full.get("response_body") or "")[:200] or "no body")
        log.warning("  repair: return TwiML with a 2xx inside 15 seconds, then "
                    "POST %s/Accounts/%s/IncomingPhoneNumbers/{PNSid}.json "
                    "VoiceFallbackUrl=https://your-app.example.com/fallback",
                    BASE, account)

    log.info("%d endpoint(s) with 11200, %d on a TwiML handler with no fallback",
             len(rows), bad)
    return 1 if bad else 0


if __name__ == "__main__":
    sys.exit(main())

/**
 * Report 11200 retrieval failures on the TwiML handlers, not the receipts.
 *
 * Read only. GET requests and nothing else: give this an API Key with read
 * access rather than the account auth token. The repair is printed, never
 * performed.
 */
const HOST = 'https://api.twilio.com';
const BASE = `${HOST}/2010-04-01`;
const MSG = 'https://messaging.twilio.com/v1';
const MONITOR = 'https://monitor.twilio.com/v1';

const PRIMARY = ['voice', 'sms', 'inbound'];

/** error_code arrives as a string on some alerts and a number on others. */
export function codeOf(alert) {
  const n = Number(alert?.error_code);
  return Number.isFinite(n) ? n : null;
}

/**
 * Reduce a URL to lowercase host plus path. Pure: applications carry a per-call
 * query string, and grouping on the raw URL turns one broken handler into forty
 * endpoints that each look survivable.
 */
export function endpoint(url) {
  let u = String(url ?? '').trim();
  for (const scheme of ['https://', 'http://']) {
    if (u.toLowerCase().startsWith(scheme)) { u = u.slice(scheme.length); break; }
  }
  u = u.split('?')[0].split('#')[0];
  if (u.split('/')[0].includes('@')) u = u.slice(u.indexOf('@') + 1);
  return u.replace(/\/+$/, '').toLowerCase();
}

/**
 * Map every configured URL to the roles it plays and what it protects. Pure:
 * the alert says which URL failed, and only this index says whether that URL is
 * a TwiML handler, a fallback or a delivery receipt.
 */
export function handlerIndex(numbers, services) {
  const idx = new Map();

  const add = (url, role, exposed = null) => {
    const e = endpoint(url);
    if (!e) return;
    const entry = idx.get(e) ?? { roles: new Set(), exposed: [] };
    entry.roles.add(role);
    if (exposed) entry.exposed.push(exposed);
    idx.set(e, entry);
  };

  for (const n of numbers) {
    const label = n.phone_number ?? n.sid ?? '?';
    const voiceFb = String(n.voice_fallback_url ?? '').trim();
    const smsFb = String(n.sms_fallback_url ?? '').trim();
    add(n.voice_url, 'voice', voiceFb ? null : `${label} voice`);
    add(n.sms_url, 'sms', smsFb ? null : `${label} sms`);
    add(voiceFb, 'fallback');
    add(smsFb, 'fallback');
    add(n.status_callback, 'status-callback');
    add(n.sms_status_callback, 'status-callback');
  }

  for (const s of services) {
    const label = s.friendly_name ?? s.sid ?? '?';
    const fb = String(s.fallback_url ?? '').trim();
    add(s.inbound_request_url, 'inbound', fb ? null : `${label} inbound`);
    add(fb, 'fallback');
    add(s.status_callback, 'status-callback');
  }

  return idx;
}

/**
 * Classify one failing endpoint. Pure, so the severity rule can be tested
 * without a network. Returns [state, detail].
 */
export function verdict(row, minAlerts = 3) {
  const roles = new Set(row.roles ?? []);
  const exposed = [...(row.exposed ?? [])];
  const n = Number(row.count ?? 0);

  if (roles.size && [...roles].every((r) => r === 'status-callback')) {
    return ['status-callback',
      `${n} failure(s) on a delivery receipt URL. That loses the receipt, not ` +
      'the call, and it is a different note with a different repair.'];
  }

  if (roles.size === 0) {
    return ['unattributed',
      `${n} failure(s) on a URL that no number and no Messaging Service ` +
      'currently points at: a TwiML App, a Studio flow, or a handler that has ' +
      'since been reconfigured.'];
  }

  const primary = PRIMARY.filter((r) => roles.has(r));
  if (primary.length === 0) {
    return ['fallback-failing',
      `${n} failure(s) on a fallback URL. The fallback is the last thing ` +
      'between a broken handler and a dropped call, and it is the thing ' +
      'returning non-2xx.'];
  }

  const where = [...primary].sort().join('/');
  if (exposed.length) {
    return ['no-safety-net',
      `${n} failure(s) on the ${where} handler for ${exposed.slice(0, 3).join(', ')}, ` +
      'which has no fallback URL. Twilio has nothing to execute, so it plays ' +
      'its own error message and hangs up, or drops the inbound message.'];
  }

  if (n < minAlerts) {
    return ['intermittent',
      `${n} failure(s) on the ${where} handler, and a fallback answered. Under ` +
      `the ${minAlerts} threshold: noise, until the rate changes.`];
  }

  return ['degraded',
    `${n} failure(s) on the ${where} handler. A fallback answered, so callers ` +
    'were served something, but not your application.'];
}

function authHeader(key, secret) {
  return `Basic ${Buffer.from(`${key}:${secret}`).toString('base64')}`;
}

async function get(auth, url, params = {}) {
  const u = new URL(url);
  for (const [k, v] of Object.entries(params)) u.searchParams.set(k, v);
  const res = await fetch(u, { headers: { Authorization: auth } });
  if (res.status === 401 || res.status === 403) {
    throw new Error(`${res.status} from Twilio: check TWILIO_ACCOUNT_SID and ` +
                    'that the API key belongs to that account with read access');
  }
  if (!res.ok) throw new Error(`${res.status} from ${u.pathname}`);
  return res.json();
}

export async function listV1(auth, url, key, limit = 10000, params = {}) {
  const out = [];
  let next = url;
  let query = { PageSize: 100, ...params };
  while (next && out.length < limit) {
    const page = await get(auth, next, query);
    out.push(...(page[key] ?? []));
    next = page.meta?.next_page_url ?? null;
    query = {};
  }
  return out.slice(0, limit);
}

export async function listNumbers(auth, account, limit = 2000) {
  let url = `${BASE}/Accounts/${account}/IncomingPhoneNumbers.json`;
  let params = { PageSize: 100 };
  const out = [];
  while (url && out.length < limit) {
    const page = await get(auth, url, params);
    out.push(...(page.incoming_phone_numbers ?? []));
    url = page.next_page_uri ? HOST + page.next_page_uri : null;
    params = {};
  }
  return out.slice(0, limit);
}

async function main() {
  const account = process.env.TWILIO_ACCOUNT_SID;
  const key = process.env.TWILIO_API_KEY;
  const secret = process.env.TWILIO_API_SECRET;
  if (!account || !key || !secret) {
    console.error('set TWILIO_ACCOUNT_SID, TWILIO_API_KEY and TWILIO_API_SECRET ' +
                  '(an API Key with read access, not the auth token)');
    process.exitCode = 2;
    return;
  }
  const auth = authHeader(key, secret);
  const days = Math.min(Number(process.env.DAYS ?? 3), 30);
  const minAlerts = Number(process.env.MIN_ALERTS ?? 3);
  const sample = process.argv.includes('--sample');

  const since = new Date(Date.now() - days * 86400000).toISOString().slice(0, 19) + 'Z';
  const alerts = (await listV1(auth, `${MONITOR}/Alerts`, 'alerts', 10000,
                               { LogLevel: 'error', StartDate: since }))
    .filter((a) => codeOf(a) === 11200);
  if (alerts.length === 0) {
    console.log(`no 11200 alerts in the last ${days} day(s)`);
    return;
  }

  const idx = handlerIndex(await listNumbers(auth, account),
                           await listV1(auth, `${MSG}/Services`, 'services', 1000));

  const rows = new Map();
  for (const a of alerts) {
    const e = endpoint(a.request_url);
    const row = rows.get(e) ??
      { endpoint: e, count: 0, sid: a.sid, roles: new Set(), exposed: [] };
    row.count += 1;
    const known = idx.get(e);
    if (known) { row.roles = known.roles; row.exposed = known.exposed; }
    rows.set(e, row);
  }

  let bad = 0;
  for (const row of [...rows.values()].sort((a, b) => b.count - a.count)) {
    const [state, detail] = verdict(row, minAlerts);
    const line = `${state.padEnd(16)} ${row.endpoint}  ${detail}`;
    if (state === 'intermittent' || state === 'status-callback') {
      console.log(line);
      continue;
    }
    bad += 1;
    console.warn(line);
    if (sample && row.sid) {
      const full = await get(auth, `${MONITOR}/Alerts/${row.sid}`);
      console.warn(`  ${full.request_method ?? 'GET'} returned: ` +
                   `${String(full.response_body ?? '').slice(0, 200) || 'no body'}`);
    }
    console.warn('  repair: return TwiML with a 2xx inside 15 seconds, then ' +
                 `POST ${BASE}/Accounts/${account}/IncomingPhoneNumbers/{PNSid}.json ` +
                 'VoiceFallbackUrl=https://your-app.example.com/fallback');
  }

  console.log(`${rows.size} endpoint(s) with 11200, ${bad} on a TwiML handler with no fallback`);
  process.exitCode = bad ? 1 : 0;
}

// Only run when invoked directly. The test file imports this module, and without
// the guard main() would run there too, fail on the missing credentials and set a
// non-zero exit code that fails the whole test file even as every test passes.
if (import.meta.url === `file://${process.argv[1]}`) {
  main().catch((err) => { console.error(err.message); process.exitCode = 2; });
}

from twilio_twiml_retrieval_audit import endpoint, handler_index, verdict

APP = "https://app.example.com/voice"
FALLBACK = "https://app.example.com/fallback"
RECEIPT = "https://app.example.com/status"


def index_for(numbers, services=()):
    return handler_index(numbers, list(services))


def test_a_status_callback_url_is_handed_to_the_other_note():
    state, detail = verdict({"count": 40, "roles": {"status-callback"}})
    assert state == "status-callback"
    assert "not the call" in detail


def test_primary_handler_with_no_fallback_is_the_dropped_call():
    idx = index_for([{"phone_number": "+15550001111", "voice_url": APP}])
    row = dict(idx[endpoint(APP)], count=5)
    state, detail = verdict(row)
    assert state == "no-safety-net"
    assert "+15550001111 voice" in detail


def test_the_same_handler_with_a_fallback_is_only_degraded():
    idx = index_for([{"phone_number": "+15550001111", "voice_url": APP,
                      "voice_fallback_url": FALLBACK}])
    state, _ = verdict(dict(idx[endpoint(APP)], count=5))
    assert state == "degraded"


def test_a_failing_fallback_is_its_own_state_not_unattributed():
    idx = index_for([{"phone_number": "+15550001111", "voice_url": APP,
                      "voice_fallback_url": FALLBACK}])
    state, detail = verdict(dict(idx[endpoint(FALLBACK)], count=2))
    assert state == "fallback-failing"
    assert "last thing" in detail


def test_an_unknown_url_is_reported_rather_than_dropped():
    state, detail = verdict({"count": 9, "roles": set()})
    assert state == "unattributed"
    assert "Studio" in detail


def test_a_few_failures_behind_a_fallback_are_under_the_threshold():
    idx = index_for([{"phone_number": "+15550001111", "sms_url": APP,
                      "sms_fallback_url": FALLBACK}])
    state, _ = verdict(dict(idx[endpoint(APP)], count=2), min_alerts=3)
    assert state == "intermittent"


def test_query_strings_do_not_split_one_handler_into_many():
    assert endpoint("https://App.Example.com/voice?CallSid=CA1") == \
        endpoint("http://app.example.com/voice/")


def test_a_service_inbound_url_counts_as_a_twiml_handler():
    idx = index_for([], [{"friendly_name": "prod", "inbound_request_url": APP,
                          "status_callback": RECEIPT}])
    state, _ = verdict(dict(idx[endpoint(APP)], count=7))
    assert state == "no-safety-net"
    assert verdict(dict(idx[endpoint(RECEIPT)], count=7))[0] == "status-callback"

import { test } from 'node:test';
import assert from 'node:assert/strict';
import { endpoint, handlerIndex, verdict } from './twilio-twiml-retrieval-audit.mjs';

const APP = 'https://app.example.com/voice';
const FALLBACK = 'https://app.example.com/fallback';
const RECEIPT = 'https://app.example.com/status';

const row = (idx, url, count, extra = {}) => ({ ...idx.get(endpoint(url)), count, ...extra });

test('a status callback url is handed to the other note', () => {
  const [state, detail] = verdict({ count: 40, roles: new Set(['status-callback']) });
  assert.equal(state, 'status-callback');
  assert.match(detail, /not the call/);
});

test('primary handler with no fallback is the dropped call', () => {
  const idx = handlerIndex([{ phone_number: '+15550001111', voice_url: APP }], []);
  const [state, detail] = verdict(row(idx, APP, 5));
  assert.equal(state, 'no-safety-net');
  assert.match(detail, /\+15550001111 voice/);
});

test('the same handler with a fallback is only degraded', () => {
  const idx = handlerIndex([{ phone_number: '+15550001111', voice_url: APP,
                              voice_fallback_url: FALLBACK }], []);
  assert.equal(verdict(row(idx, APP, 5))[0], 'degraded');
});

test('a failing fallback is its own state, not unattributed', () => {
  const idx = handlerIndex([{ phone_number: '+15550001111', voice_url: APP,
                              voice_fallback_url: FALLBACK }], []);
  assert.equal(verdict(row(idx, FALLBACK, 2))[0], 'fallback-failing');
});

test('an unknown url is reported rather than dropped', () => {
  const [state, detail] = verdict({ count: 9, roles: new Set() });
  assert.equal(state, 'unattributed');
  assert.match(detail, /Studio/);
});

test('a few failures behind a fallback are under the threshold', () => {
  const idx = handlerIndex([{ phone_number: '+15550001111', sms_url: APP,
                              sms_fallback_url: FALLBACK }], []);
  assert.equal(verdict(row(idx, APP, 2), 3)[0], 'intermittent');
});

test('query strings do not split one handler into many', () => {
  assert.equal(endpoint('https://App.Example.com/voice?CallSid=CA1'),
               endpoint('http://app.example.com/voice/'));
});

test('a service inbound url counts as a twiml handler', () => {
  const idx = handlerIndex([], [{ friendly_name: 'prod', inbound_request_url: APP,
                                  status_callback: RECEIPT }]);
  assert.equal(verdict(row(idx, APP, 7))[0], 'no-safety-net');
  assert.equal(verdict(row(idx, RECEIPT, 7))[0], 'status-callback');
});