13224: Twilio refuses the number your Dial verb asked for

"""Report Twilio 13224 alerts and say why each Dial destination was refused.

Read only. GET requests and nothing else: give this an API Key with read access
rather than the account auth token. The repair is printed, never performed,
because this script holds a credential to an account that can place calls and
spend money.
"""
import argparse
import datetime as dt
import logging
import os
import sys

import requests

logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s")
log = logging.getLogger("twilio_dial_target_audit")

HOST = "https://api.twilio.com"
BASE = HOST + "/2010-04-01"
MONITOR = "https://monitor.twilio.com/v1"

UNSUPPORTED = 13224

# Ranges that are premium, shared cost or special service by international
# allocation rather than by national convention. The table is deliberately
# short. Every country also has its own premium ranges, and a table of all of
# them is a maintenance project you will lose; Lookups settles the rest.
REFUSED_PREFIXES = (
    ("+979", "ITU international premium rate service"),
    ("+808", "ITU international shared cost service"),
    ("+882", "ITU international networks"),
    ("+883", "ITU international networks"),
    ("+881", "global mobile satellite system"),
    ("+870", "Inmarsat single network access code"),
    ("+4470", "UK personal numbering, forwarded at premium cost"),
    ("+449", "UK premium rate"),
    ("+1900", "North American premium rate"),
)

# Directions whose call record carries the destination that was dialled. An
# inbound call does not: its `to` is your own number.
OUTBOUND = ("outbound-api", "outbound-dial", "trunking")


def e164_digits(to):
    """The digits of a strictly E.164 destination, or an empty string.

    Strict deliberately. A plus, then one to fifteen digits, and nothing else:
    no spaces, no brackets, no dashes, no leading zero after the plus.
    Normalising the punctuation away here would destroy the evidence, because a
    column of national-format numbers going straight into the Dial noun is the
    single most common cause of this error.
    """
    v = str(to or "").strip()
    if not v.startswith("+"):
        return ""
    digits = v[1:]
    if not digits.isdigit() or not 1 <= len(digits) <= 15:
        return ""
    return digits


def refused_range(to):
    """The allocation a destination falls in, or an empty string.

    Longest prefix wins, so +4470 is reported as personal numbering rather than
    as UK premium rate.
    """
    v = str(to or "").strip()
    best, label = "", ""
    for prefix, name in REFUSED_PREFIXES:
        if v.startswith(prefix) and len(prefix) > len(best):
            best, label = prefix, name
    return label


def verdict(call):
    """Explain one 13224 from the call it was raised against.

    Pure, so the rules can be tested without a network. `call` is the Call
    resource the alert's resource_sid resolved to. Returns (state, detail).
    """
    to = str(call.get("to") or "").strip()
    direction = str(call.get("direction") or "").strip().lower()

    if not to:
        return ("no-destination",
                "the call record has no `to`, so there is nothing to classify. "
                "Read the single alert for the request variables.")

    if direction and direction not in OUTBOUND:
        return ("target-not-on-record",
                "direction is %s, so `to` (%s) is the number the caller dialled "
                "and not the destination that was refused. The dial target is "
                "in the request variables, which are populated only on GET "
                "/v1/Alerts/{AlertSid}." % (direction, to))

    low = to.lower()
    if low.startswith("sip:") or low.startswith("sips:") or low.startswith("client:"):
        return ("non-pstn",
                "%s is not a PSTN destination, so this refusal is about a "
                "different Dial noun and E.164 has nothing to do with it." % to)

    if not to.startswith("+"):
        return ("not-e164",
                "%s has no leading plus, so Twilio cannot tell which country it "
                "belongs to. This is national format arriving straight from a "
                "column that predates E.164." % to)

    digits = e164_digits(to)
    if not digits:
        return ("malformed",
                "%s starts with a plus but is not digits after it, or runs past "
                "the fifteen digit E.164 ceiling. The punctuation is the "
                "finding: the value was never normalised." % to)

    if len(digits) < 8:
        return ("too-short",
                "%s carries only %d digits, which is shorter than a full "
                "international destination. This is usually an internal "
                "extension dialled as though it were a phone number."
                % (to, len(digits)))

    allocation = refused_range(to)
    if allocation:
        return ("refused-range",
                "%s is in the %s range. It is well formed and it is unsupported, "
                "which is the other half of the error text: Twilio will not "
                "terminate on it, today or ever." % (to, allocation))

    return ("unallocated",
            "%s is shaped correctly and is outside the ranges this table knows, "
            "so the number itself does not exist: an unassigned area code, a "
            "country code that was never allocated, or a digit lost in "
            "transcription. Lookups v2 will report valid false." % to)


def get(session, url, **params):
    r = session.get(url, params=params, timeout=30)
    if r.status_code in (401, 403):
        raise SystemExit("%d from Twilio: check TWILIO_ACCOUNT_SID and that the "
                         "API key belongs to that account with read access"
                         % r.status_code)
    r.raise_for_status()
    return r.json()


def list_alerts(session, since, limit, log_level):
    """Page the Monitor alerts at one log level. next_page_url is absolute."""
    url = MONITOR + "/Alerts"
    params = {"LogLevel": log_level, "StartDate": since, "PageSize": 1000}
    out = []
    while url and len(out) < limit:
        page = get(session, url, **params)
        out.extend(page.get("alerts", []))
        url = (page.get("meta") or {}).get("next_page_url")
        params = {}
    return out[:limit]


def sweep_alerts(session, since, limit, levels):
    """Both log levels, merged on sid.

    Several of the 132xx Dial attribute errors are logged at warning rather than
    error. A sweep that reads only the error level reports a clean account while
    the legs keep failing, which is why this takes a list of levels at all.
    """
    seen = {}
    for level in levels:
        for a in list_alerts(session, since, limit, level):
            seen.setdefault(a.get("sid"), a)
    return list(seen.values())


def main():
    ap = argparse.ArgumentParser(description=__doc__)
    ap.add_argument("--days", type=int, default=7,
                    help="how far back to sweep (alerts are retained 30 days)")
    ap.add_argument("--max-alerts", type=int, default=10000,
                    help="stop after this many alerts per log level")
    ap.add_argument("--errors-only", action="store_true",
                    help="skip the warning level, which will under-report")
    ap.add_argument("--alert-detail", action="store_true",
                    help="one extra GET per inbound case for the request variables")
    args = ap.parse_args()

    account = os.environ.get("TWILIO_ACCOUNT_SID")
    key = os.environ.get("TWILIO_API_KEY")
    secret = os.environ.get("TWILIO_API_SECRET")
    if not (account and key and secret):
        log.error("set TWILIO_ACCOUNT_SID, TWILIO_API_KEY and TWILIO_API_SECRET "
                  "(an API Key with read access, not the auth token)")
        return 2

    session = requests.Session()
    session.auth = (key, secret)

    days = min(args.days, 30)
    since = (dt.date.today() - dt.timedelta(days=days)).isoformat()
    levels = ["error"] if args.errors_only else ["error", "warning"]

    alerts = sweep_alerts(session, since, args.max_alerts, levels)
    hits = [a for a in alerts
            if str(a.get("error_code") or "").strip() == str(UNSUPPORTED)]
    if not hits:
        log.info("0 alert(s) with error_code %d in the last %d day(s)",
                 UNSUPPORTED, days)
        return 0

    calls = {}
    counts = {}
    for a in hits:
        sid = str(a.get("resource_sid") or "")
        if not sid.startswith("CA"):
            log.warning("13224 alert %s has no call sid to resolve", a.get("sid"))
            continue
        if sid not in calls:
            calls[sid] = get(session, "%s/Accounts/%s/Calls/%s.json"
                             % (BASE, account, sid))
        state, detail = verdict(calls[sid])
        counts[state] = counts.get(state, 0) + 1
        log.warning("%-21s %s  %s", state, sid, detail)
        if state == "target-not-on-record" and args.alert_detail:
            one = get(session, "%s/Alerts/%s" % (MONITOR, a.get("sid")))
            log.warning("  alert_text: %s", one.get("alert_text"))

    log.warning("%d alert(s) with error_code %d across %d call(s): %s",
                len(hits), UNSUPPORTED, len(calls),
                ", ".join("%s=%d" % kv for kv in sorted(counts.items())))
    log.warning("  repair: normalise the destination column to E.164 where it "
                "is stored, then validate with GET "
                "https://lookups.twilio.com/v2/PhoneNumbers/{E164} and keep "
                "only valid == true")
    log.warning("  repair: exclude premium and special service ranges from the "
                "dial list; they are refused every time")
    return 1


if __name__ == "__main__":
    sys.exit(main())

/**
 * Report Twilio 13224 alerts and say why each Dial destination was refused.
 *
 * Read only. GET requests and nothing else: give this an API Key with read
 * access rather than the account auth token. The repair is printed, never
 * performed.
 */
const HOST = 'https://api.twilio.com';
const BASE = `${HOST}/2010-04-01`;
const MONITOR = 'https://monitor.twilio.com/v1';

const UNSUPPORTED = 13224;

// Premium, shared cost and special service ranges by international allocation.
// Deliberately short: national premium ranges are a table you will lose track
// of, and Lookups settles the rest.
const REFUSED_PREFIXES = [
  ['+979', 'ITU international premium rate service'],
  ['+808', 'ITU international shared cost service'],
  ['+882', 'ITU international networks'],
  ['+883', 'ITU international networks'],
  ['+881', 'global mobile satellite system'],
  ['+870', 'Inmarsat single network access code'],
  ['+4470', 'UK personal numbering, forwarded at premium cost'],
  ['+449', 'UK premium rate'],
  ['+1900', 'North American premium rate'],
];

const OUTBOUND = ['outbound-api', 'outbound-dial', 'trunking'];

/**
 * The digits of a strictly E.164 destination, or an empty string. Strict on
 * purpose: cleaning the punctuation here would destroy the evidence.
 */
export function e164Digits(to) {
  const v = String(to ?? '').trim();
  if (!v.startsWith('+')) return '';
  const digits = v.slice(1);
  if (!/^[0-9]+$/.test(digits) || digits.length > 15) return '';
  return digits;
}

/** The allocation a destination falls in, or an empty string. Longest wins. */
export function refusedRange(to) {
  const v = String(to ?? '').trim();
  let best = '';
  let label = '';
  for (const [prefix, name] of REFUSED_PREFIXES) {
    if (v.startsWith(prefix) && prefix.length > best.length) {
      best = prefix;
      label = name;
    }
  }
  return label;
}

/**
 * Explain one 13224 from the call it was raised against. Pure. Returns
 * [state, detail].
 */
export function verdict(call) {
  const to = String(call.to ?? '').trim();
  const direction = String(call.direction ?? '').trim().toLowerCase();

  if (!to) {
    return ['no-destination',
      'the call record has no `to`, so there is nothing to classify. Read the ' +
      'single alert for the request variables.'];
  }

  if (direction && !OUTBOUND.includes(direction)) {
    return ['target-not-on-record',
      `direction is ${direction}, so \`to\` (${to}) is the number the caller ` +
      'dialled and not the destination that was refused. The dial target is in ' +
      'the request variables, which are populated only on GET ' +
      '/v1/Alerts/{AlertSid}.'];
  }

  const low = to.toLowerCase();
  if (low.startsWith('sip:') || low.startsWith('sips:') || low.startsWith('client:')) {
    return ['non-pstn',
      `${to} is not a PSTN destination, so this refusal is about a different ` +
      'Dial noun and E.164 has nothing to do with it.'];
  }

  if (!to.startsWith('+')) {
    return ['not-e164',
      `${to} has no leading plus, so Twilio cannot tell which country it ` +
      'belongs to. This is national format arriving straight from a column ' +
      'that predates E.164.'];
  }

  const digits = e164Digits(to);
  if (!digits) {
    return ['malformed',
      `${to} starts with a plus but is not digits after it, or runs past the ` +
      'fifteen digit E.164 ceiling. The punctuation is the finding: the value ' +
      'was never normalised.'];
  }

  if (digits.length < 8) {
    return ['too-short',
      `${to} carries only ${digits.length} digits, which is shorter than a ` +
      'full international destination. This is usually an internal extension ' +
      'dialled as though it were a phone number.'];
  }

  const allocation = refusedRange(to);
  if (allocation) {
    return ['refused-range',
      `${to} is in the ${allocation} range. It is well formed and it is ` +
      'unsupported, which is the other half of the error text: Twilio will not ' +
      'terminate on it, today or ever.'];
  }

  return ['unallocated',
    `${to} is shaped correctly and is outside the ranges this table knows, so ` +
    'the number itself does not exist: an unassigned area code, a country code ' +
    'that was never allocated, or a digit lost in transcription. Lookups v2 ' +
    'will report valid false.'];
}

function authHeader(key, secret) {
  return `Basic ${Buffer.from(`${key}:${secret}`).toString('base64')}`;
}

async function get(auth, url, params = {}) {
  const u = new URL(url);
  for (const [k, v] of Object.entries(params)) u.searchParams.set(k, v);
  const res = await fetch(u, { headers: { Authorization: auth } });
  if (res.status === 401 || res.status === 403) {
    throw new Error(`${res.status} from Twilio: check TWILIO_ACCOUNT_SID and ` +
                    'that the API key belongs to that account with read access');
  }
  if (!res.ok) throw new Error(`${res.status} from ${u.pathname}`);
  return res.json();
}

async function listAlerts(auth, since, limit, logLevel) {
  let url = `${MONITOR}/Alerts`;
  let params = { LogLevel: logLevel, StartDate: since, PageSize: 1000 };
  const out = [];
  while (url && out.length < limit) {
    const page = await get(auth, url, params);
    out.push(...(page.alerts ?? []));
    url = page.meta?.next_page_url ?? null;
    params = {};
  }
  return out.slice(0, limit);
}

/** Both log levels, merged on sid. Some 132xx errors are logged as warnings. */
export async function sweepAlerts(auth, since, limit, levels) {
  const seen = new Map();
  for (const level of levels) {
    for (const a of await listAlerts(auth, since, limit, level)) {
      if (!seen.has(a.sid)) seen.set(a.sid, a);
    }
  }
  return [...seen.values()];
}

async function main() {
  const account = process.env.TWILIO_ACCOUNT_SID;
  const key = process.env.TWILIO_API_KEY;
  const secret = process.env.TWILIO_API_SECRET;
  if (!account || !key || !secret) {
    console.error('set TWILIO_ACCOUNT_SID, TWILIO_API_KEY and TWILIO_API_SECRET ' +
                  '(an API Key with read access, not the auth token)');
    process.exitCode = 2;
    return;
  }
  const auth = authHeader(key, secret);
  const arg = (name, fallback) => {
    const i = process.argv.indexOf(name);
    return i === -1 ? fallback : Number(process.argv[i + 1]);
  };
  const days = Math.min(arg('--days', 7), 30);
  const detail = process.argv.includes('--alert-detail');
  const levels = process.argv.includes('--errors-only') ? ['error'] : ['error', 'warning'];

  const since = new Date(Date.now() - days * 86400000).toISOString().slice(0, 10);
  const alerts = await sweepAlerts(auth, since, 10000, levels);
  const hits = alerts.filter((a) => String(a.error_code ?? '').trim() === String(UNSUPPORTED));
  if (hits.length === 0) {
    console.log(`0 alert(s) with error_code ${UNSUPPORTED} in the last ${days} day(s)`);
    return;
  }

  const calls = new Map();
  const counts = new Map();
  for (const a of hits) {
    const sid = String(a.resource_sid ?? '');
    if (!sid.startsWith('CA')) {
      console.warn(`13224 alert ${a.sid} has no call sid to resolve`);
      continue;
    }
    if (!calls.has(sid)) {
      calls.set(sid, await get(auth, `${BASE}/Accounts/${account}/Calls/${sid}.json`));
    }
    const [state, why] = verdict(calls.get(sid));
    counts.set(state, (counts.get(state) ?? 0) + 1);
    console.warn(`${state.padEnd(21)} ${sid}  ${why}`);
    if (state === 'target-not-on-record' && detail) {
      const one = await get(auth, `${MONITOR}/Alerts/${a.sid}`);
      console.warn(`  alert_text: ${one.alert_text}`);
    }
  }

  const summary = [...counts.entries()].sort().map(([k, v]) => `${k}=${v}`).join(', ');
  console.warn(`${hits.length} alert(s) with error_code ${UNSUPPORTED} across ` +
               `${calls.size} call(s): ${summary}`);
  console.warn('  repair: normalise the destination column to E.164 where it is ' +
               'stored, then validate with GET ' +
               'https://lookups.twilio.com/v2/PhoneNumbers/{E164} and keep only ' +
               'valid == true');
  console.warn('  repair: exclude premium and special service ranges from the ' +
               'dial list; they are refused every time');
  process.exitCode = 1;
}

// Only run when invoked directly. The test file imports this module, and without
// the guard main() would run there too, fail on the missing credentials and set a
// non-zero exit code that fails the whole test file even as every test passes.
if (import.meta.url === `file://${process.argv[1]}`) {
  main().catch((err) => { console.error(err.message); process.exitCode = 2; });
}

from twilio_dial_target_audit import e164_digits, refused_range, verdict


def test_national_format_is_the_common_cause():
    state, detail = verdict({"to": "01614960000", "direction": "outbound-api"})
    assert state == "not-e164"
    assert "predates E.164" in detail


def test_punctuated_number_is_malformed_rather_than_tidied():
    # Cleaning it here would hide the thing the application should have done.
    state, _ = verdict({"to": "+44 161 496 0000", "direction": "outbound-api"})
    assert state == "malformed"


def test_inbound_call_does_not_carry_the_dial_target():
    state, detail = verdict({"to": "+441614960000", "direction": "inbound"})
    assert state == "target-not-on-record"
    assert "AlertSid" in detail


def test_premium_range_is_unsupported_not_invalid():
    state, detail = verdict({"to": "+19005551234", "direction": "outbound-api"})
    assert state == "refused-range"
    assert "North American premium rate" in detail


def test_longest_prefix_wins_over_the_shorter_one():
    assert refused_range("+447012345678") == \
        "UK personal numbering, forwarded at premium cost"
    assert refused_range("+449001234567") == "UK premium rate"


def test_extension_dialled_as_a_number_is_too_short():
    state, _ = verdict({"to": "+4021", "direction": "outbound-dial"})
    assert state == "too-short"


def test_well_formed_unknown_number_points_at_lookups():
    state, detail = verdict({"to": "+15005550001", "direction": "outbound-api"})
    assert state == "unallocated"
    assert "valid false" in detail


def test_e164_digits_is_strict_about_the_ceiling_and_the_plus():
    assert e164_digits("+441614960000") == "441614960000"
    assert e164_digits("441614960000") == ""
    assert e164_digits("+1234567890123456") == ""

import { test } from 'node:test';
import assert from 'node:assert/strict';
import { e164Digits, refusedRange, verdict } from './twilio-dial-target-audit.mjs';

test('national format is the common cause', () => {
  const [state, detail] = verdict({ to: '01614960000', direction: 'outbound-api' });
  assert.equal(state, 'not-e164');
  assert.match(detail, /predates E\.164/);
});

test('punctuated number is malformed rather than tidied', () => {
  assert.equal(verdict({ to: '+44 161 496 0000', direction: 'outbound-api' })[0],
               'malformed');
});

test('inbound call does not carry the dial target', () => {
  const [state, detail] = verdict({ to: '+441614960000', direction: 'inbound' });
  assert.equal(state, 'target-not-on-record');
  assert.match(detail, /AlertSid/);
});

test('premium range is unsupported not invalid', () => {
  const [state, detail] = verdict({ to: '+19005551234', direction: 'outbound-api' });
  assert.equal(state, 'refused-range');
  assert.match(detail, /North American premium rate/);
});

test('longest prefix wins over the shorter one', () => {
  assert.equal(refusedRange('+447012345678'),
               'UK personal numbering, forwarded at premium cost');
  assert.equal(refusedRange('+449001234567'), 'UK premium rate');
});

test('extension dialled as a number is too short', () => {
  assert.equal(verdict({ to: '+4021', direction: 'outbound-dial' })[0], 'too-short');
});

test('well formed unknown number points at lookups', () => {
  const [state, detail] = verdict({ to: '+15005550001', direction: 'outbound-api' });
  assert.equal(state, 'unallocated');
  assert.match(detail, /valid false/);
});

test('e164Digits is strict about the ceiling and the plus', () => {
  assert.equal(e164Digits('+441614960000'), '441614960000');
  assert.equal(e164Digits('441614960000'), '');
  assert.equal(e164Digits('+1234567890123456'), '');
});