SMS to a landline fails with 30006 and retrying never helps

"""Report SMS destinations that can never receive a message: 30006 and 21614.

Read only. GET requests and nothing else: give this an API Key with read access
rather than the account auth token. The repair is printed, never performed,
because this script holds a credential to an account that can spend money.
"""
import argparse
import datetime as dt
import logging
import os
import sys

import requests

logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s")
log = logging.getLogger("twilio_landline_audit")

HOST = "https://api.twilio.com"
BASE = HOST + "/2010-04-01"
LOOKUPS = "https://lookups.twilio.com/v2/PhoneNumbers"

UNDELIVERABLE = 30006   # undelivered, after the segment was billed
NOT_MOBILE = 21614      # rejected at request time, never billed

NO_SMS = ("landline", "fixedvoip")


def error_code(message):
    """Read error_code as an integer, or None. It is null on healthy messages
    and a number on failed ones; a string comparison finds nothing."""
    raw = message.get("error_code")
    if raw is None or raw == "":
        return None
    try:
        return int(raw)
    except (TypeError, ValueError):
        return None


def tally(messages):
    """Group failures by destination, keeping the two codes apart.

    Pure, so the counting can be tested without a network. 30006 was billed and
    21614 was not, and a report that adds them together loses the only number
    anyone will ask you for.
    """
    out = {}
    for m in messages:
        if str(m.get("direction") or "").startswith("inbound"):
            continue
        code = error_code(m)
        if code not in (UNDELIVERABLE, NOT_MOBILE):
            continue
        row = out.setdefault(str(m.get("to") or "unknown"),
                             {"attempts": 0, "undelivered": 0, "rejected": 0,
                              "sids": []})
        row["attempts"] += 1
        if code == UNDELIVERABLE:
            row["undelivered"] += 1
        else:
            row["rejected"] += 1
        if len(row["sids"]) < 3:
            row["sids"].append(m.get("sid"))
    return out


def describe(record):
    """Say which of the two failures this destination produced, and at what
    cost. Pure."""
    parts = []
    if record.get("undelivered"):
        parts.append("%d undelivered with 30006 and billed"
                     % record["undelivered"])
    if record.get("rejected"):
        parts.append("%d rejected at request time with 21614 and not billed"
                     % record["rejected"])
    return " and ".join(parts) if parts else "no refused attempts"


def verdict(record, line_type=None):
    """Classify one destination. `line_type` is line_type_intelligence.type from
    Lookup when it was fetched, and None when it was not.

    Pure, so the distinction that matters here can be tested without spending a
    lookup. Returns (state, detail).
    """
    failed = int(record.get("undelivered") or 0) + int(record.get("rejected") or 0)
    if not failed:
        return ("clean", "%d attempt(s), none refused" % (record.get("attempts") or 0))

    told = describe(record)
    kind = str(line_type or "").strip()

    if kind.lower() in NO_SMS:
        return ("landline",
                "Lookup says %s, which cannot receive SMS at any price: %s. "
                "Retrying never helps." % (kind, told))

    if kind.lower() == "mobile":
        return ("sender-cannot-reach",
                "Lookup says mobile, so this is not a landline: %s. The handset "
                "is fine and the sending route cannot reach that carrier, which "
                "is what a short code with no long code fallback looks like."
                % told)

    if kind and kind.lower() != "unknown":
        return ("not-sms-capable",
                "Lookup says %s, which is not an SMS capable line: %s."
                % (kind, told))

    if failed == 1:
        return ("one-off",
                "a single failure and no line type: %s. Confirm with Lookup "
                "before dropping the contact." % told)

    return ("undeliverable",
            "%d refused attempt(s) with no line type: %s. Treat it as permanent "
            "and confirm with Lookup Line Type Intelligence." % (failed, told))


def get(session, url, **params):
    r = session.get(url, params=params, timeout=30)
    if r.status_code in (401, 403):
        raise SystemExit("%d from Twilio: check TWILIO_ACCOUNT_SID and that the "
                         "API key belongs to that account with read access"
                         % r.status_code)
    r.raise_for_status()
    return r.json()


def list_messages(session, account, since, limit):
    """Page Messages.json. No ErrorCode filter exists on this resource, so the
    window and the cap are the only bounds available."""
    url = "%s/Accounts/%s/Messages.json" % (BASE, account)
    params = {"PageSize": 1000, "DateSent>=": since}
    out = []
    while url and len(out) < limit:
        page = get(session, url, **params)
        out.extend(page.get("messages", []))
        nxt = page.get("next_page_uri")
        url, params = (HOST + nxt) if nxt else None, {}
    return out[:limit]


def line_type(session, e164):
    """One billed Lookup. A 404 means the number is not valid at all, which is
    an answer rather than an error."""
    r = session.get("%s/%s" % (LOOKUPS, e164),
                    params={"Fields": "line_type_intelligence"}, timeout=30)
    if r.status_code == 404:
        return "invalid"
    if r.status_code in (401, 403):
        raise SystemExit("%d from Lookups: the API key needs read access to "
                         "Lookup as well" % r.status_code)
    r.raise_for_status()
    body = r.json()
    if body.get("valid") is False:
        return "invalid"
    return (body.get("line_type_intelligence") or {}).get("type")


def main():
    ap = argparse.ArgumentParser(description=__doc__)
    ap.add_argument("--days", type=int, default=30,
                    help="how far back to read the Messages list")
    ap.add_argument("--max-messages", type=int, default=20000,
                    help="stop paging after this many messages")
    ap.add_argument("--confirm-with-lookup", action="store_true",
                    help="one billed Lookup per flagged destination")
    ap.add_argument("--max-lookups", type=int, default=50,
                    help="hard cap on billed lookups per run")
    args = ap.parse_args()

    account = os.environ.get("TWILIO_ACCOUNT_SID")
    key = os.environ.get("TWILIO_API_KEY")
    secret = os.environ.get("TWILIO_API_SECRET")
    if not (account and key and secret):
        log.error("set TWILIO_ACCOUNT_SID, TWILIO_API_KEY and TWILIO_API_SECRET "
                  "(an API Key with read access, not the auth token)")
        return 2

    session = requests.Session()
    session.auth = (key, secret)

    since = (dt.date.today() - dt.timedelta(days=args.days)).isoformat()
    messages = list_messages(session, account, since, args.max_messages)
    destinations = tally(messages)
    if not destinations:
        log.info("no 30006 or 21614 failures since %s", since)
        return 0

    spent = 0
    bad = 0
    for number, record in sorted(destinations.items()):
        kind = None
        if args.confirm_with_lookup and spent < args.max_lookups:
            kind = line_type(session, number)
            spent += 1
        state, detail = verdict(record, kind)
        line = "%-20s %s  %s" % (state, number, detail)
        if state == "clean":
            log.info(line)
            continue
        bad += 1
        log.warning(line)
        log.warning("  message sids: %s", ", ".join(str(s) for s in record["sids"]))
        if state == "sender-cannot-reach":
            log.warning("  repair: add a long code sender to the Messaging "
                        "Service pool with POST %s/Services/{ServiceSid}"
                        "/PhoneNumbers PhoneNumberSid=PN...",
                        "https://messaging.twilio.com/v1")
        else:
            log.warning("  repair: suppress %s in your own database and gate new "
                        "numbers at capture time with GET %s/{E164}"
                        "?Fields=line_type_intelligence", number, LOOKUPS)

    log.info("%d destination(s) over %d day(s), %d still being retried",
             len(destinations), args.days, bad)
    return 1 if bad else 0


if __name__ == "__main__":
    sys.exit(main())

/**
 * Report SMS destinations that can never receive a message: 30006 and 21614.
 *
 * Read only. GET requests and nothing else: give this an API Key with read
 * access rather than the account auth token. The repair is printed, never
 * performed.
 */
const HOST = 'https://api.twilio.com';
const BASE = `${HOST}/2010-04-01`;
const LOOKUPS = 'https://lookups.twilio.com/v2/PhoneNumbers';
const MSG = 'https://messaging.twilio.com/v1';

const UNDELIVERABLE = 30006;  // undelivered, after the segment was billed
const NOT_MOBILE = 21614;     // rejected at request time, never billed

const NO_SMS = ['landline', 'fixedvoip'];

/** Read error_code as a number, or null. A string comparison finds nothing. */
export function errorCode(message) {
  const raw = message.error_code;
  if (raw === null || raw === undefined || raw === '') return null;
  const n = Number(raw);
  return Number.isFinite(n) ? n : null;
}

/**
 * Group failures by destination, keeping the two codes apart: 30006 was billed
 * and 21614 was not. Pure, so the counting can be tested without a network.
 */
export function tally(messages) {
  const out = new Map();
  for (const m of messages) {
    if (String(m.direction ?? '').startsWith('inbound')) continue;
    const code = errorCode(m);
    if (code !== UNDELIVERABLE && code !== NOT_MOBILE) continue;
    const k = String(m.to ?? 'unknown');
    if (!out.has(k)) out.set(k, { attempts: 0, undelivered: 0, rejected: 0, sids: [] });
    const row = out.get(k);
    row.attempts += 1;
    if (code === UNDELIVERABLE) row.undelivered += 1; else row.rejected += 1;
    if (row.sids.length < 3) row.sids.push(m.sid);
  }
  return out;
}

/** Say which failures this destination produced, and at what cost. Pure. */
export function describe(record) {
  const parts = [];
  if (record.undelivered) {
    parts.push(`${record.undelivered} undelivered with 30006 and billed`);
  }
  if (record.rejected) {
    parts.push(`${record.rejected} rejected at request time with 21614 and not billed`);
  }
  return parts.length ? parts.join(' and ') : 'no refused attempts';
}

/**
 * Classify one destination. `lineType` is line_type_intelligence.type from
 * Lookup when it was fetched, and null when it was not. Pure, so the
 * distinction that matters can be tested without spending a lookup.
 * Returns [state, detail].
 */
export function verdict(record, lineType = null) {
  const failed = Number(record.undelivered ?? 0) + Number(record.rejected ?? 0);
  if (!failed) return ['clean', `${record.attempts ?? 0} attempt(s), none refused`];

  const told = describe(record);
  const kind = String(lineType ?? '').trim();

  if (NO_SMS.includes(kind.toLowerCase())) {
    return ['landline',
      `Lookup says ${kind}, which cannot receive SMS at any price: ${told}. ` +
      'Retrying never helps.'];
  }

  if (kind.toLowerCase() === 'mobile') {
    return ['sender-cannot-reach',
      `Lookup says mobile, so this is not a landline: ${told}. The handset is ` +
      'fine and the sending route cannot reach that carrier, which is what a ' +
      'short code with no long code fallback looks like.'];
  }

  if (kind && kind.toLowerCase() !== 'unknown') {
    return ['not-sms-capable',
      `Lookup says ${kind}, which is not an SMS capable line: ${told}.`];
  }

  if (failed === 1) {
    return ['one-off',
      `a single failure and no line type: ${told}. Confirm with Lookup before ` +
      'dropping the contact.'];
  }

  return ['undeliverable',
    `${failed} refused attempt(s) with no line type: ${told}. Treat it as ` +
    'permanent and confirm with Lookup Line Type Intelligence.'];
}

function authHeader(key, secret) {
  return `Basic ${Buffer.from(`${key}:${secret}`).toString('base64')}`;
}

async function get(auth, url, params = {}) {
  const u = new URL(url);
  for (const [k, v] of Object.entries(params)) u.searchParams.set(k, v);
  const res = await fetch(u, { headers: { Authorization: auth } });
  if (res.status === 401 || res.status === 403) {
    throw new Error(`${res.status} from Twilio: check TWILIO_ACCOUNT_SID and ` +
                    'that the API key belongs to that account with read access');
  }
  if (!res.ok) throw new Error(`${res.status} from ${u.pathname}`);
  return res.json();
}

export async function listMessages(auth, account, since, limit = 20000) {
  let url = `${BASE}/Accounts/${account}/Messages.json`;
  let params = { PageSize: 1000, 'DateSent>=': since };
  const out = [];
  while (url && out.length < limit) {
    const page = await get(auth, url, params);
    out.push(...(page.messages ?? []));
    url = page.next_page_uri ? HOST + page.next_page_uri : null;
    params = {};
  }
  return out.slice(0, limit);
}

async function lineType(auth, e164) {
  const u = new URL(`${LOOKUPS}/${e164}`);
  u.searchParams.set('Fields', 'line_type_intelligence');
  const res = await fetch(u, { headers: { Authorization: auth } });
  if (res.status === 404) return 'invalid';
  if (!res.ok) throw new Error(`${res.status} from Lookups for ${e164}`);
  const body = await res.json();
  if (body.valid === false) return 'invalid';
  return body.line_type_intelligence?.type ?? null;
}

async function main() {
  const account = process.env.TWILIO_ACCOUNT_SID;
  const key = process.env.TWILIO_API_KEY;
  const secret = process.env.TWILIO_API_SECRET;
  if (!account || !key || !secret) {
    console.error('set TWILIO_ACCOUNT_SID, TWILIO_API_KEY and TWILIO_API_SECRET ' +
                  '(an API Key with read access, not the auth token)');
    process.exitCode = 2;
    return;
  }
  const auth = authHeader(key, secret);

  const days = Number(process.argv.includes('--days')
    ? process.argv[process.argv.indexOf('--days') + 1] : 30) || 30;
  const confirm = process.argv.includes('--confirm-with-lookup');
  const maxLookups = 50;
  const since = new Date(Date.now() - days * 86400000).toISOString().slice(0, 10);

  const destinations = tally(await listMessages(auth, account, since));
  if (destinations.size === 0) {
    console.log(`no 30006 or 21614 failures since ${since}`);
    return;
  }

  let spent = 0;
  let bad = 0;
  for (const [number, record] of [...destinations.entries()].sort()) {
    let kind = null;
    if (confirm && spent < maxLookups) { kind = await lineType(auth, number); spent += 1; }
    const [state, detail] = verdict(record, kind);
    const line = `${state.padEnd(20)} ${number}  ${detail}`;
    if (state === 'clean') { console.log(line); continue; }
    bad += 1;
    console.warn(line);
    console.warn(`  message sids: ${record.sids.join(', ')}`);
    if (state === 'sender-cannot-reach') {
      console.warn('  repair: add a long code sender to the Messaging Service ' +
                   `pool with POST ${MSG}/Services/{ServiceSid}/PhoneNumbers ` +
                   'PhoneNumberSid=PN...');
    } else {
      console.warn(`  repair: suppress ${number} in your own database and gate ` +
                   `new numbers at capture time with GET ${LOOKUPS}/{E164}` +
                   '?Fields=line_type_intelligence');
    }
  }

  console.log(`${destinations.size} destination(s) over ${days} day(s), ${bad} ` +
              'still being retried');
  process.exitCode = bad ? 1 : 0;
}

// Only run when invoked directly. The test file imports this module, and without
// the guard main() would run there too, fail on the missing credentials and set a
// non-zero exit code that fails the whole test file even as every test passes.
if (import.meta.url === `file://${process.argv[1]}`) {
  main().catch((err) => { console.error(err.message); process.exitCode = 2; });
}

from twilio_landline_audit import describe, tally, verdict

DESK = "+15551230000"


def failure(sid, code, to=DESK):
    return {"sid": sid, "direction": "outbound-api", "to": to,
            "status": "undelivered", "error_code": code}


def test_the_two_codes_are_counted_separately():
    rows = tally([failure("SM1", 30006), failure("SM2", 21614),
                  failure("SM3", 30006), failure("SM4", 30007)])
    assert rows[DESK]["undelivered"] == 2
    assert rows[DESK]["rejected"] == 1
    assert rows[DESK]["attempts"] == 3   # 30007 belongs to a different report


def test_describe_says_which_half_was_billed():
    told = describe({"undelivered": 2, "rejected": 1})
    assert "billed" in told
    assert "not billed" in told


def test_lookup_landline_is_permanent():
    state, detail = verdict({"undelivered": 4}, "landline")
    assert state == "landline"
    assert "Retrying never helps" in detail


def test_fixed_voip_is_treated_like_a_landline():
    assert verdict({"undelivered": 2}, "fixedVoip")[0] == "landline"


def test_a_mobile_that_keeps_failing_is_the_senders_problem():
    state, detail = verdict({"undelivered": 6}, "mobile")
    assert state == "sender-cannot-reach"
    assert "short code" in detail


def test_no_lookup_and_one_failure_is_not_yet_a_verdict():
    state, detail = verdict({"rejected": 1})
    assert state == "one-off"
    assert "Confirm with Lookup" in detail


def test_no_lookup_and_repeated_failures_is_treated_as_permanent():
    state, detail = verdict({"undelivered": 5})
    assert state == "undeliverable"
    assert "5 refused" in detail


def test_an_unknown_line_type_does_not_pretend_to_know():
    assert verdict({"undelivered": 5}, "unknown")[0] == "undeliverable"
    assert verdict({"undelivered": 5}, "invalid")[0] == "not-sms-capable"
    assert verdict({"attempts": 3})[0] == "clean"

import { test } from 'node:test';
import assert from 'node:assert/strict';
import { describe as told, tally, verdict } from './twilio-landline-audit.mjs';

const DESK = '+15551230000';
const failure = (sid, code, to = DESK) => ({
  sid, direction: 'outbound-api', to, status: 'undelivered', error_code: code,
});

test('the two codes are counted separately', () => {
  const rows = tally([failure('SM1', 30006), failure('SM2', 21614),
                      failure('SM3', 30006), failure('SM4', 30007)]);
  const row = rows.get(DESK);
  assert.equal(row.undelivered, 2);
  assert.equal(row.rejected, 1);
  assert.equal(row.attempts, 3);  // 30007 belongs to a different report
});

test('describe says which half was billed', () => {
  const line = told({ undelivered: 2, rejected: 1 });
  assert.match(line, /and billed/);
  assert.match(line, /not billed/);
});

test('lookup landline is permanent', () => {
  const [state, detail] = verdict({ undelivered: 4 }, 'landline');
  assert.equal(state, 'landline');
  assert.match(detail, /Retrying never helps/);
});

test('fixed voip is treated like a landline', () => {
  assert.equal(verdict({ undelivered: 2 }, 'fixedVoip')[0], 'landline');
});

test('a mobile that keeps failing is the sender problem', () => {
  const [state, detail] = verdict({ undelivered: 6 }, 'mobile');
  assert.equal(state, 'sender-cannot-reach');
  assert.match(detail, /short code/);
});

test('no lookup and one failure is not yet a verdict', () => {
  const [state, detail] = verdict({ rejected: 1 });
  assert.equal(state, 'one-off');
  assert.match(detail, /Confirm with Lookup/);
});

test('no lookup and repeated failures is treated as permanent', () => {
  const [state, detail] = verdict({ undelivered: 5 });
  assert.equal(state, 'undeliverable');
  assert.match(detail, /5 refused/);
});

test('an unknown line type does not pretend to know', () => {
  assert.equal(verdict({ undelivered: 5 }, 'unknown')[0], 'undeliverable');
  assert.equal(verdict({ undelivered: 5 }, 'invalid')[0], 'not-sms-capable');
  assert.equal(verdict({ attempts: 3 })[0], 'clean');
});