error 30003 is a handset that is off, not a dead number

"""Report Twilio 30003 failures, split into unreachable handsets and a blocked sender.

Read only. GET requests and nothing else: give this an API Key with read access
rather than the account auth token. The repair is printed, never performed,
because this script holds a credential to an account that can send messages and
spend money.
"""
import argparse
import datetime as dt
import logging
import os
import sys

import requests

logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s")
log = logging.getLogger("twilio_unreachable_handset_audit")

HOST = "https://api.twilio.com"
BASE = HOST + "/2010-04-01"

UNREACHABLE = 30003


def error_code(message):
    """Read error_code as an integer, or None.

    It is null on every healthy message and a number on failed ones, but it
    arrives as a string often enough to matter. Comparing the raw value against
    30003 is the mistake that makes this audit report nothing on an account that
    is full of findings.
    """
    raw = message.get("error_code")
    if raw is None or raw == "":
        return None
    try:
        return int(raw)
    except (TypeError, ValueError):
        return None


def group(messages):
    """Bucket 30003 twice over: by recipient and by sender.

    Pure, so both grouping rules can be tested without a network. Recipients with
    no 30003 are dropped at the end; they are only tracked along the way so that
    a failing number's delivered count is available, which is what separates a
    flaky handset from a number that has never once taken a message.

    Returns (recipients, senders).
    """
    recipients = {}
    senders = {}
    touched = {}

    for m in messages:
        if str(m.get("direction") or "").startswith("inbound"):
            continue

        sender = m.get("messaging_service_sid") or m.get("from") or "unknown sender"
        stats = senders.setdefault(sender, {"total": 0, "failed": 0,
                                            "recipients": 0, "sids": []})
        stats["total"] += 1

        to = m.get("to") or "unknown recipient"
        row = recipients.setdefault(to, {"hits": 0, "delivered": 0, "sids": []})
        if str(m.get("status") or "").lower() == "delivered":
            row["delivered"] += 1

        if error_code(m) == UNREACHABLE:
            row["hits"] += 1
            if len(row["sids"]) < 3:
                row["sids"].append(m.get("sid"))
            stats["failed"] += 1
            if len(stats["sids"]) < 3:
                stats["sids"].append(m.get("sid"))
            touched.setdefault(sender, set()).add(to)

    for sender, tos in touched.items():
        senders[sender]["recipients"] = len(tos)

    return ({k: v for k, v in recipients.items() if v["hits"]}, senders)


def recipient_verdict(row):
    """Classify one recipient's 30003 history. Pure, so the rule is testable.

    Returns (state, detail).
    """
    hits = int(row.get("hits") or 0)
    delivered = int(row.get("delivered") or 0)

    if hits <= 1:
        return ("transient",
                "one 30003 and %d delivered. Powered off, out of coverage or "
                "roaming: retry once after a delay and expect it to arrive."
                % delivered)

    if delivered:
        return ("flaky",
                "%d unreachable, %d delivered in the same window. This number "
                "does take SMS, just not every time: back the retries off, do "
                "not drop it." % (hits, delivered))

    return ("never-reached",
            "%d unreachable and not one delivery, ever. Stop retrying and run "
            "Lookup line type intelligence: a number that has never accepted a "
            "message is usually not a mobile." % hits)


def sender_verdict(stats, min_failed=3):
    """Classify one sender's 30003 rate. Pure, so the thresholds are visible.

    Returns (state, detail).
    """
    total = int(stats.get("total") or 0)
    failed = int(stats.get("failed") or 0)
    distinct = int(stats.get("recipients") or 0)

    if not failed:
        return ("clean", "%d message(s), no 30003" % total)

    rate = (failed / total) if total else 1.0

    if failed < min_failed:
        return ("isolated",
                "%d of %d unreachable (%.1f%%). Too few to read anything into: "
                "handsets are off all the time." % (failed, total, rate * 100))

    if distinct and failed / distinct >= 3:
        return ("dead-numbers",
                "%d failures over only %d recipient(s). The failures pile onto a "
                "handful of numbers, so this is list decay rather than anything "
                "wrong with the sender." % (failed, distinct))

    if rate >= 0.2:
        return ("sender-blocked",
                "%d of %d unreachable (%.1f%%) across %d recipient(s). No carrier "
                "switches off a fifth of its subscribers at once: at this spread "
                "30003 is masking a block on the sender itself."
                % (failed, total, rate * 100, distinct))

    return ("handsets",
            "%d of %d unreachable (%.1f%%) across %d recipient(s). Thin and spread "
            "out, which is what genuine handset unreachability looks like: one "
            "retry each." % (failed, total, rate * 100, distinct))


def get(session, url, **params):
    r = session.get(url, params=params, timeout=30)
    if r.status_code in (401, 403):
        raise SystemExit("%d from Twilio: check TWILIO_ACCOUNT_SID and that the "
                         "API key belongs to that account with read access"
                         % r.status_code)
    r.raise_for_status()
    return r.json()


def list_messages(session, account, since, limit):
    """Page Messages.json. There is no Status or ErrorCode filter on this
    resource, so the date window and the page cap are the only bounds available."""
    url = "%s/Accounts/%s/Messages.json" % (BASE, account)
    params = {"PageSize": 1000, "DateSent>=": since}
    out = []
    while url and len(out) < limit:
        page = get(session, url, **params)
        out.extend(page.get("messages", []))
        nxt = page.get("next_page_uri")
        url, params = (HOST + nxt) if nxt else None, {}
    return out[:limit]


def main():
    ap = argparse.ArgumentParser(description=__doc__)
    ap.add_argument("--days", type=int, default=7,
                    help="how far back to read the Messages list")
    ap.add_argument("--max-messages", type=int, default=20000,
                    help="stop paging after this many messages")
    ap.add_argument("--min-failed", type=int, default=3,
                    help="fewer than this on one sender is reported as isolated")
    args = ap.parse_args()

    account = os.environ.get("TWILIO_ACCOUNT_SID")
    key = os.environ.get("TWILIO_API_KEY")
    secret = os.environ.get("TWILIO_API_SECRET")
    if not (account and key and secret):
        log.error("set TWILIO_ACCOUNT_SID, TWILIO_API_KEY and TWILIO_API_SECRET "
                  "(an API Key with read access, not the auth token)")
        return 2

    session = requests.Session()
    session.auth = (key, secret)

    since = (dt.date.today() - dt.timedelta(days=args.days)).isoformat()
    messages = list_messages(session, account, since, args.max_messages)
    if not messages:
        log.info("no messages sent since %s", since)
        return 0

    recipients, senders = group(messages)

    bad = 0
    for sender, stats in sorted(senders.items()):
        state, detail = sender_verdict(stats, args.min_failed)
        line = "%-14s %s  %s" % (state, sender, detail)
        if state in ("clean", "isolated", "handsets"):
            log.info(line)
            continue
        bad += 1
        log.warning(line)
        log.warning("  message sids: %s", ", ".join(str(s) for s in stats["sids"]))
        if state == "sender-blocked":
            log.warning("  repair: no API call fixes this. Send those SIDs to "
                        "Twilio Support and ask whether the sender is blocked "
                        "on the destination carrier.")
        else:
            log.warning("  repair: check each repeat offender with GET "
                        "https://lookups.twilio.com/v2/PhoneNumbers/{E164}"
                        "?Fields=line_type_intelligence and drop anything whose "
                        "line_type_intelligence.type is not mobile.")

    for to, row in sorted(recipients.items()):
        state, detail = recipient_verdict(row)
        if state == "transient":
            continue
        log.warning("%-14s %s  %s", state, to, detail)

    log.info("%d sender(s), %d recipient(s) with a 30003, %d sender-level problem(s)",
             len(senders), len(recipients), bad)
    return 1 if bad else 0


if __name__ == "__main__":
    sys.exit(main())

/**
 * Report Twilio 30003 failures, split into unreachable handsets and a blocked sender.
 *
 * Read only. GET requests and nothing else: give this an API Key with read
 * access rather than the account auth token. The repair is printed, never
 * performed.
 */
const HOST = 'https://api.twilio.com';
const BASE = `${HOST}/2010-04-01`;

const UNREACHABLE = 30003;

/**
 * Read error_code as a number, or null. It is null on healthy messages and a
 * number on failed ones, but it arrives as a string often enough that comparing
 * the raw value against 30003 is how this audit reports nothing on an account
 * full of findings.
 */
export function errorCode(message) {
  const raw = message.error_code;
  if (raw === null || raw === undefined || raw === '') return null;
  const n = Number(raw);
  return Number.isFinite(n) ? n : null;
}

/**
 * Bucket 30003 twice over: by recipient and by sender. Pure, so both grouping
 * rules can be tested without a network. Recipients with no 30003 are dropped at
 * the end; they are tracked along the way only so a failing number's delivered
 * count is available. Returns { recipients, senders }.
 */
export function group(messages) {
  const recipients = new Map();
  const senders = new Map();
  const touched = new Map();

  for (const m of messages) {
    if (String(m.direction ?? '').startsWith('inbound')) continue;

    const sender = m.messaging_service_sid || m.from || 'unknown sender';
    if (!senders.has(sender)) {
      senders.set(sender, { total: 0, failed: 0, recipients: 0, sids: [] });
    }
    const stats = senders.get(sender);
    stats.total += 1;

    const to = m.to || 'unknown recipient';
    if (!recipients.has(to)) recipients.set(to, { hits: 0, delivered: 0, sids: [] });
    const row = recipients.get(to);
    if (String(m.status ?? '').toLowerCase() === 'delivered') row.delivered += 1;

    if (errorCode(m) === UNREACHABLE) {
      row.hits += 1;
      if (row.sids.length < 3) row.sids.push(m.sid);
      stats.failed += 1;
      if (stats.sids.length < 3) stats.sids.push(m.sid);
      if (!touched.has(sender)) touched.set(sender, new Set());
      touched.get(sender).add(to);
    }
  }

  for (const [sender, tos] of touched) senders.get(sender).recipients = tos.size;
  for (const [to, row] of [...recipients]) if (!row.hits) recipients.delete(to);

  return { recipients, senders };
}

/**
 * Classify one recipient's 30003 history. Pure. Returns [state, detail].
 */
export function recipientVerdict(row) {
  const hits = Number(row.hits ?? 0);
  const delivered = Number(row.delivered ?? 0);

  if (hits <= 1) {
    return ['transient',
      `one 30003 and ${delivered} delivered. Powered off, out of coverage or ` +
      'roaming: retry once after a delay and expect it to arrive.'];
  }

  if (delivered) {
    return ['flaky',
      `${hits} unreachable, ${delivered} delivered in the same window. This ` +
      'number does take SMS, just not every time: back the retries off, do not ' +
      'drop it.'];
  }

  return ['never-reached',
    `${hits} unreachable and not one delivery, ever. Stop retrying and run ` +
    'Lookup line type intelligence: a number that has never accepted a message ' +
    'is usually not a mobile.'];
}

/**
 * Classify one sender's 30003 rate. Pure, so the thresholds are visible and
 * testable. Returns [state, detail].
 */
export function senderVerdict(stats, minFailed = 3) {
  const total = Number(stats.total ?? 0);
  const failed = Number(stats.failed ?? 0);
  const distinct = Number(stats.recipients ?? 0);

  if (!failed) return ['clean', `${total} message(s), no 30003`];

  const rate = total ? failed / total : 1;
  const pct = (rate * 100).toFixed(1);

  if (failed < minFailed) {
    return ['isolated',
      `${failed} of ${total} unreachable (${pct}%). Too few to read anything ` +
      'into: handsets are off all the time.'];
  }

  if (distinct && failed / distinct >= 3) {
    return ['dead-numbers',
      `${failed} failures over only ${distinct} recipient(s). The failures pile ` +
      'onto a handful of numbers, so this is list decay rather than anything ' +
      'wrong with the sender.'];
  }

  if (rate >= 0.2) {
    return ['sender-blocked',
      `${failed} of ${total} unreachable (${pct}%) across ${distinct} ` +
      'recipient(s). No carrier switches off a fifth of its subscribers at ' +
      'once: at this spread 30003 is masking a block on the sender itself.'];
  }

  return ['handsets',
    `${failed} of ${total} unreachable (${pct}%) across ${distinct} recipient(s). ` +
    'Thin and spread out, which is what genuine handset unreachability looks ' +
    'like: one retry each.'];
}

function authHeader(key, secret) {
  return `Basic ${Buffer.from(`${key}:${secret}`).toString('base64')}`;
}

async function get(auth, url, params = {}) {
  const u = new URL(url);
  for (const [k, v] of Object.entries(params)) u.searchParams.set(k, v);
  const res = await fetch(u, { headers: { Authorization: auth } });
  if (res.status === 401 || res.status === 403) {
    throw new Error(`${res.status} from Twilio: check TWILIO_ACCOUNT_SID and ` +
                    'that the API key belongs to that account with read access');
  }
  if (!res.ok) throw new Error(`${res.status} from ${u.pathname}`);
  return res.json();
}

export async function listMessages(auth, account, since, limit = 20000) {
  let url = `${BASE}/Accounts/${account}/Messages.json`;
  let params = { PageSize: 1000, 'DateSent>=': since };
  const out = [];
  while (url && out.length < limit) {
    const page = await get(auth, url, params);
    out.push(...(page.messages ?? []));
    url = page.next_page_uri ? HOST + page.next_page_uri : null;
    params = {};
  }
  return out.slice(0, limit);
}

async function main() {
  const account = process.env.TWILIO_ACCOUNT_SID;
  const key = process.env.TWILIO_API_KEY;
  const secret = process.env.TWILIO_API_SECRET;
  if (!account || !key || !secret) {
    console.error('set TWILIO_ACCOUNT_SID, TWILIO_API_KEY and TWILIO_API_SECRET ' +
                  '(an API Key with read access, not the auth token)');
    process.exitCode = 2;
    return;
  }
  const auth = authHeader(key, secret);

  const days = Number(process.argv.includes('--days')
    ? process.argv[process.argv.indexOf('--days') + 1] : 7) || 7;
  const since = new Date(Date.now() - days * 86400000).toISOString().slice(0, 10);

  const messages = await listMessages(auth, account, since);
  if (messages.length === 0) {
    console.log(`no messages sent since ${since}`);
    return;
  }

  const { recipients, senders } = group(messages);

  let bad = 0;
  for (const [sender, stats] of [...senders.entries()].sort()) {
    const [state, detail] = senderVerdict(stats);
    const line = `${state.padEnd(14)} ${sender}  ${detail}`;
    if (state === 'clean' || state === 'isolated' || state === 'handsets') {
      console.log(line);
      continue;
    }
    bad += 1;
    console.warn(line);
    console.warn(`  message sids: ${stats.sids.join(', ')}`);
    if (state === 'sender-blocked') {
      console.warn('  repair: no API call fixes this. Send those SIDs to Twilio ' +
                   'Support and ask whether the sender is blocked on the ' +
                   'destination carrier.');
    } else {
      console.warn('  repair: check each repeat offender with GET ' +
                   'https://lookups.twilio.com/v2/PhoneNumbers/{E164}' +
                   '?Fields=line_type_intelligence and drop anything whose ' +
                   'line_type_intelligence.type is not mobile.');
    }
  }

  for (const [to, row] of [...recipients.entries()].sort()) {
    const [state, detail] = recipientVerdict(row);
    if (state === 'transient') continue;
    console.warn(`${state.padEnd(14)} ${to}  ${detail}`);
  }

  console.log(`${senders.size} sender(s), ${recipients.size} recipient(s) with a ` +
              `30003, ${bad} sender-level problem(s)`);
  process.exitCode = bad ? 1 : 0;
}

// Only run when invoked directly. The test file imports this module, and without
// the guard main() would run there too, fail on the missing credentials and set a
// non-zero exit code that fails the whole test file even as every test passes.
if (import.meta.url === `file://${process.argv[1]}`) {
  main().catch((err) => { console.error(err.message); process.exitCode = 2; });
}

from twilio_unreachable_handset_audit import (error_code, group,
                                              recipient_verdict, sender_verdict)


def unreachable(sid, to="+15557770001", sender="+15550001111"):
    return {"sid": sid, "to": to, "from": sender, "status": "undelivered",
            "error_code": 30003, "direction": "outbound-api"}


def delivered(sid, to="+15557770001", sender="+15550001111"):
    return {"sid": sid, "to": to, "from": sender, "status": "delivered",
            "error_code": None, "direction": "outbound-api"}


def test_error_code_reads_strings_and_numbers_the_same():
    assert error_code({"error_code": 30003}) == 30003
    assert error_code({"error_code": "30003"}) == 30003
    assert error_code({"error_code": None}) is None
    assert error_code({}) is None


def test_group_drops_recipients_that_never_failed():
    recipients, senders = group([unreachable("SM1"), delivered("SM2", to="+15557770002")])
    assert set(recipients) == {"+15557770001"}
    assert senders["+15550001111"]["total"] == 2
    assert senders["+15550001111"]["failed"] == 1


def test_group_counts_distinct_recipients_per_sender():
    msgs = [unreachable("SM%d" % i, to="+1555777%04d" % i) for i in range(5)]
    _, senders = group(msgs)
    assert senders["+15550001111"]["recipients"] == 5


def test_group_prefers_the_messaging_service_over_the_from_number():
    m = unreachable("SM1")
    m["messaging_service_sid"] = "MG1"
    _, senders = group([m])
    assert set(senders) == {"MG1"}


def test_group_ignores_inbound_messages():
    recipients, senders = group([{"sid": "SM1", "to": "+15550001111",
                                  "direction": "inbound", "status": "received"}])
    assert recipients == {}
    assert senders == {}


def test_one_failure_is_transient():
    state, detail = recipient_verdict({"hits": 1, "delivered": 0})
    assert state == "transient"
    assert "retry once" in detail


def test_a_number_that_also_delivers_is_flaky_and_stays_on_the_list():
    state, detail = recipient_verdict({"hits": 4, "delivered": 2})
    assert state == "flaky"
    assert "do not drop it" in detail


def test_repeated_failures_with_no_delivery_ever_go_to_lookup():
    state, detail = recipient_verdict({"hits": 4, "delivered": 0})
    assert state == "never-reached"
    assert "Lookup" in detail


def test_no_failures_is_a_clean_sender():
    state, detail = sender_verdict({"total": 900, "failed": 0})
    assert state == "clean"
    assert "900" in detail


def test_two_failures_are_too_few_to_mean_anything():
    state, _ = sender_verdict({"total": 4, "failed": 2, "recipients": 1})
    assert state == "isolated"


def test_many_failures_over_few_recipients_is_list_decay():
    # Same failure count as the blocked-sender case below; the spread is the
    # only thing that distinguishes them.
    state, detail = sender_verdict({"total": 100, "failed": 12, "recipients": 3})
    assert state == "dead-numbers"
    assert "list decay" in detail


def test_the_same_failures_spread_wide_is_a_blocked_sender():
    state, detail = sender_verdict({"total": 100, "failed": 30, "recipients": 30})
    assert state == "sender-blocked"
    assert "fifth" in detail


def test_a_thin_wide_spread_is_ordinary_handsets():
    state, detail = sender_verdict({"total": 500, "failed": 5, "recipients": 5})
    assert state == "handsets"
    assert "one retry each" in detail

import { test } from 'node:test';
import assert from 'node:assert/strict';
import { errorCode, group, recipientVerdict, senderVerdict }
  from './twilio-unreachable-handset-audit.mjs';

const unreachable = (sid, to = '+15557770001', sender = '+15550001111') => ({
  sid, to, from: sender, status: 'undelivered', error_code: 30003,
  direction: 'outbound-api',
});

const delivered = (sid, to = '+15557770001', sender = '+15550001111') => ({
  sid, to, from: sender, status: 'delivered', error_code: null,
  direction: 'outbound-api',
});

test('error code reads strings and numbers the same', () => {
  assert.equal(errorCode({ error_code: 30003 }), 30003);
  assert.equal(errorCode({ error_code: '30003' }), 30003);
  assert.equal(errorCode({ error_code: null }), null);
  assert.equal(errorCode({}), null);
});

test('group drops recipients that never failed', () => {
  const { recipients, senders } = group([
    unreachable('SM1'), delivered('SM2', '+15557770002')]);
  assert.deepEqual([...recipients.keys()], ['+15557770001']);
  assert.equal(senders.get('+15550001111').total, 2);
  assert.equal(senders.get('+15550001111').failed, 1);
});

test('group counts distinct recipients per sender', () => {
  const msgs = [0, 1, 2, 3, 4].map((i) => unreachable(`SM${i}`, `+1555777000${i}`));
  const { senders } = group(msgs);
  assert.equal(senders.get('+15550001111').recipients, 5);
});

test('group prefers the messaging service over the from number', () => {
  const m = { ...unreachable('SM1'), messaging_service_sid: 'MG1' };
  const { senders } = group([m]);
  assert.deepEqual([...senders.keys()], ['MG1']);
});

test('group ignores inbound messages', () => {
  const { recipients, senders } = group([
    { sid: 'SM1', to: '+15550001111', direction: 'inbound', status: 'received' }]);
  assert.equal(recipients.size, 0);
  assert.equal(senders.size, 0);
});

test('one failure is transient', () => {
  const [state, detail] = recipientVerdict({ hits: 1, delivered: 0 });
  assert.equal(state, 'transient');
  assert.match(detail, /retry once/);
});

test('a number that also delivers is flaky and stays on the list', () => {
  const [state, detail] = recipientVerdict({ hits: 4, delivered: 2 });
  assert.equal(state, 'flaky');
  assert.match(detail, /do not drop it/);
});

test('repeated failures with no delivery ever go to Lookup', () => {
  const [state, detail] = recipientVerdict({ hits: 4, delivered: 0 });
  assert.equal(state, 'never-reached');
  assert.match(detail, /Lookup/);
});

test('no failures is a clean sender', () => {
  const [state, detail] = senderVerdict({ total: 900, failed: 0 });
  assert.equal(state, 'clean');
  assert.match(detail, /900/);
});

test('two failures are too few to mean anything', () => {
  const [state] = senderVerdict({ total: 4, failed: 2, recipients: 1 });
  assert.equal(state, 'isolated');
});

test('many failures over few recipients is list decay', () => {
  const [state, detail] = senderVerdict({ total: 100, failed: 12, recipients: 3 });
  assert.equal(state, 'dead-numbers');
  assert.match(detail, /list decay/);
});

test('the same failures spread wide is a blocked sender', () => {
  const [state, detail] = senderVerdict({ total: 100, failed: 30, recipients: 30 });
  assert.equal(state, 'sender-blocked');
  assert.match(detail, /fifth/);
});

test('a thin wide spread is ordinary handsets', () => {
  const [state, detail] = senderVerdict({ total: 500, failed: 5, recipients: 5 });
  assert.equal(state, 'handsets');
  assert.match(detail, /one retry each/);
});