an alphanumeric sender ID is unregistered where you send

"""Report alphanumeric sender IDs rejected by the destination carrier.

Read only. GET requests and nothing else: give this an API Key with read access
rather than the account auth token. The repair is printed, never performed,
because this script holds a credential to an account that can send messages and
spend money.
"""
import argparse
import datetime as dt
import logging
import os
import sys

import requests

logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s")
log = logging.getLogger("twilio_alpha_sender_audit")

HOST = "https://api.twilio.com"
BASE = HOST + "/2010-04-01"
MESSAGING = "https://messaging.twilio.com/v1"

# Rejected outright by the destination carrier for a sender it does not know.
BLOCKING = (30040, 30041)
# The warning-level sibling. Logged below error level, so an alert sweep
# filtered to LogLevel=error never shows it.
WARNING = 30018

# Countries that mandate pre-registration of alphanumeric sender IDs. The list
# grows; it is used to explain a finding, never to decide one, because the
# decision is made by what the traffic did.
REGISTRATION_REQUIRED = {"91": "India", "966": "Saudi Arabia", "971": "the UAE",
                         "84": "Vietnam", "880": "Bangladesh", "94": "Sri Lanka",
                         "977": "Nepal", "998": "Uzbekistan"}

DIAL_CODES = {
    "1", "7", "20", "27", "30", "31", "32", "33", "34", "36", "39", "40", "43",
    "44", "45", "46", "47", "48", "49", "51", "52", "54", "55", "56", "57",
    "58", "60", "61", "62", "63", "64", "65", "66", "81", "82", "84", "86",
    "90", "91", "92", "94", "212", "213", "234", "254", "351", "353", "358",
    "380", "420", "421", "852", "880", "886", "966", "971", "972", "977", "998",
}


def error_code(message):
    """Read error_code as an integer, or None. Some exports hand it back as a
    string, and comparing the raw value finds nothing on a broken account."""
    raw = message.get("error_code")
    if raw is None or raw == "":
        return None
    try:
        return int(raw)
    except (TypeError, ValueError):
        return None


def sender_kind(value):
    """Classify a From value as e164, short-code or alphanumeric.

    Alphanumeric sender IDs are the only ones this audit is about, and the one
    thing that distinguishes them in a message row is that they are not a
    number. A digit string short enough to be a short code is not one either.
    """
    raw = str(value or "").strip()
    if not raw:
        return "unknown"
    if raw.startswith("+"):
        return "e164"
    if raw.isdigit():
        return "short-code" if len(raw) <= 8 else "e164"
    return "alphanumeric"


def dial_code(to):
    """Longest matching country calling code for an E.164 destination, or None.

    Registration is granted per country, so the destination country is half of
    the key this audit groups on.
    """
    raw = str(to or "").strip()
    if not raw.startswith("+"):
        return None
    digits = "".join(c for c in raw[1:] if c.isdigit())
    for size in (3, 2, 1):
        if digits[:size] in DIAL_CODES:
            return digits[:size]
    return None


def tally(messages):
    """Bucket outbound alphanumeric-sender messages by sender and destination.

    Pure, so the grouping rule can be tested without a network. The key is the
    pair, not the sender: a sender ID registered in one country and not in the
    next is the normal case, and a per-sender total hides it.
    """
    out = {}
    for m in messages:
        if str(m.get("direction") or "").startswith("inbound"):
            continue
        sender = str(m.get("from") or "").strip()
        if sender_kind(sender) != "alphanumeric":
            continue
        code = dial_code(m.get("to"))
        row = out.setdefault((sender, code),
                             {"sender": sender, "code": code, "total": 0,
                              "blocked": 0, "warned": 0, "accepted": 0, "sids": []})
        row["total"] += 1
        err = error_code(m)
        if err in BLOCKING:
            row["blocked"] += 1
            if len(row["sids"]) < 3:
                row["sids"].append(m.get("sid"))
        elif err == WARNING:
            row["warned"] += 1
        else:
            row["accepted"] += 1
    return out


def verdict(row, configured=None):
    """Classify one sender-and-country pair.

    `configured` is the set of alpha_sender strings attached to the account's
    Messaging Services, or None when those were not read. It is not a list of
    what a regulator has approved, because no API returns that; it is only good
    enough to catch the case mismatch, which is the cheap fix worth separating
    from the slow one.

    Pure. Returns (state, detail).
    """
    sender = str(row.get("sender") or "")
    code = row.get("code")
    where = "+%s" % code if code else "an unresolved destination"
    if code in REGISTRATION_REQUIRED:
        where = REGISTRATION_REQUIRED[code]
    total = int(row.get("total") or 0)
    blocked = int(row.get("blocked") or 0)
    warned = int(row.get("warned") or 0)

    known = set(configured or ())
    exact = sender in known
    folded = {s.casefold() for s in known}
    near = (not exact) and sender.casefold() in folded

    if blocked:
        if near:
            return ("case-mismatch",
                    "%d of %d to %s rejected with 30040/30041, and '%s' differs "
                    "from a configured sender only in case. Sender IDs are "
                    "matched byte for byte, so this is a change in your sending "
                    "code, not a registration." % (blocked, total, where, sender))
        return ("unregistered",
                "%d of %d to %s rejected with 30040/30041. The destination "
                "carrier requires this sender to be pre-registered there; the "
                "API accepted every one of these because it cannot know that."
                % (blocked, total, where))

    if warned:
        return ("warned",
                "%d of %d to %s carry 30018. That is the warning-level sibling "
                "of 30041 and it is below the error threshold most alert sweeps "
                "use, so this is the notice you would otherwise miss."
                % (warned, total, where))

    if configured is not None and not exact:
        return ("not-in-pool",
                "%d message(s) to %s delivering from '%s', which is not attached "
                "to any Messaging Service. It works today, but nothing on the "
                "account records that this string is a sender of yours."
                % (total, where, sender))

    return ("delivering", "%d message(s) to %s, none rejected" % (total, where))


def get(session, url, **params):
    r = session.get(url, params=params, timeout=30)
    if r.status_code in (401, 403):
        raise SystemExit("%d from Twilio: check TWILIO_ACCOUNT_SID and that the "
                         "API key belongs to that account with read access"
                         % r.status_code)
    r.raise_for_status()
    return r.json()


def list_messages(session, account, since, limit):
    """Page Messages.json. No Status or ErrorCode filter exists on this
    resource, so the window and the page cap are the only bounds."""
    url = "%s/Accounts/%s/Messages.json" % (BASE, account)
    params = {"PageSize": 1000, "DateSent>=": since}
    out = []
    while url and len(out) < limit:
        page = get(session, url, **params)
        out.extend(page.get("messages", []))
        nxt = page.get("next_page_uri")
        url, params = (HOST + nxt) if nxt else None, {}
    return out[:limit]


def configured_senders(session):
    """Every alpha_sender attached to every Messaging Service, exactly as
    returned. Not a registration list: no API returns one."""
    out = {}
    services = get(session, "%s/Services" % MESSAGING, PageSize=100).get("services", [])
    for svc in services:
        sid = svc.get("sid")
        page = get(session, "%s/Services/%s/AlphaSenders" % (MESSAGING, sid),
                   PageSize=100)
        for alpha in page.get("alpha_senders", []):
            out[str(alpha.get("alpha_sender") or "")] = sid
    return out


def main():
    ap = argparse.ArgumentParser(description=__doc__)
    ap.add_argument("--days", type=int, default=7,
                    help="how far back to read the message list")
    ap.add_argument("--max-messages", type=int, default=20000,
                    help="stop paging after this many messages")
    ap.add_argument("--skip-services", action="store_true",
                    help="do not read the Messaging Services, which disables the "
                         "case-mismatch comparison")
    args = ap.parse_args()

    account = os.environ.get("TWILIO_ACCOUNT_SID")
    key = os.environ.get("TWILIO_API_KEY")
    secret = os.environ.get("TWILIO_API_SECRET")
    if not (account and key and secret):
        log.error("set TWILIO_ACCOUNT_SID, TWILIO_API_KEY and TWILIO_API_SECRET "
                  "(an API Key with read access, not the auth token)")
        return 2

    session = requests.Session()
    session.auth = (key, secret)

    since = (dt.date.today() - dt.timedelta(days=args.days)).isoformat()
    messages = list_messages(session, account, since, args.max_messages)
    pairs = tally(messages)
    if not pairs:
        log.info("no messages from an alphanumeric sender since %s", since)
        return 0

    owners = {} if args.skip_services else configured_senders(session)
    configured = None if args.skip_services else set(owners)

    bad = 0
    for _, row in sorted(pairs.items(), key=lambda kv: (kv[0][0], str(kv[0][1]))):
        state, detail = verdict(row, configured)
        line = "%-14s %-12s %s" % (state, row["sender"][:12], detail)
        if state == "delivering":
            log.info(line)
            continue
        bad += 1
        log.warning(line)
        if row["sids"]:
            log.warning("  message sids: %s", ", ".join(str(s) for s in row["sids"]))
        if state == "case-mismatch":
            match = [s for s in owners if s.casefold() == row["sender"].casefold()]
            log.warning("  repair: send From='%s', the string already configured "
                        "on service %s. No registration is needed for that.",
                        match[0], owners.get(match[0], "?"))
        elif state == "unregistered":
            log.warning("  repair: register '%s' for this country at Console -> "
                        "Messaging -> Senders -> Alphanumeric Sender IDs, then "
                        "attach it with a create call on %s/Services/{ServiceSid}"
                        "/AlphaSenders. Until it is approved, route this country "
                        "through a long code.", row["sender"], MESSAGING)
        else:
            log.warning("  repair: attach '%s' to the Messaging Service that "
                        "should own it, so the account records it as a sender.",
                        row["sender"])

    log.info("%d sender/destination pair(s), %d rejected by the destination "
             "carrier", len(pairs), bad)
    return 1 if bad else 0


if __name__ == "__main__":
    sys.exit(main())

/**
 * Report alphanumeric sender IDs rejected by the destination carrier.
 *
 * Read only. GET requests and nothing else: give this an API Key with read
 * access rather than the account auth token. The repair is printed, never
 * performed.
 */
const HOST = 'https://api.twilio.com';
const BASE = `${HOST}/2010-04-01`;
const MESSAGING = 'https://messaging.twilio.com/v1';

/** Rejected outright by the destination carrier for a sender it does not know. */
const BLOCKING = [30040, 30041];
/** The warning-level sibling, below the threshold most alert sweeps use. */
const WARNING = 30018;

/** Countries that mandate pre-registration. Used to explain, never to decide. */
const REGISTRATION_REQUIRED = {
  91: 'India', 966: 'Saudi Arabia', 971: 'the UAE', 84: 'Vietnam',
  880: 'Bangladesh', 94: 'Sri Lanka', 977: 'Nepal', 998: 'Uzbekistan',
};

const DIAL_CODES = new Set([
  '1', '7', '20', '27', '30', '31', '32', '33', '34', '36', '39', '40', '43',
  '44', '45', '46', '47', '48', '49', '51', '52', '54', '55', '56', '57', '58',
  '60', '61', '62', '63', '64', '65', '66', '81', '82', '84', '86', '90', '91',
  '92', '94', '212', '213', '234', '254', '351', '353', '358', '380', '420',
  '421', '852', '880', '886', '966', '971', '972', '977', '998',
]);

/** Read error_code as a number, or null. */
export function errorCode(message) {
  const raw = message.error_code;
  if (raw === null || raw === undefined || raw === '') return null;
  const n = Number(raw);
  return Number.isFinite(n) ? n : null;
}

/**
 * Classify a From value as e164, short-code or alphanumeric. Alphanumeric
 * senders are the only rows this audit is about.
 */
export function senderKind(value) {
  const raw = String(value ?? '').trim();
  if (!raw) return 'unknown';
  if (raw.startsWith('+')) return 'e164';
  if (/^\d+$/.test(raw)) return raw.length <= 8 ? 'short-code' : 'e164';
  return 'alphanumeric';
}

/** Longest matching country calling code for an E.164 destination, or null. */
export function dialCode(to) {
  const raw = String(to ?? '').trim();
  if (!raw.startsWith('+')) return null;
  const digits = raw.slice(1).replace(/\D/g, '');
  for (const size of [3, 2, 1]) {
    const head = digits.slice(0, size);
    if (DIAL_CODES.has(head)) return head;
  }
  return null;
}

/**
 * Bucket outbound alphanumeric-sender messages by sender and destination. Pure,
 * so the grouping rule can be tested without a network. The key is the pair:
 * registration is granted for one sender in one country.
 */
export function tally(messages) {
  const out = new Map();
  for (const m of messages) {
    if (String(m.direction ?? '').startsWith('inbound')) continue;
    const sender = String(m.from ?? '').trim();
    if (senderKind(sender) !== 'alphanumeric') continue;
    const code = dialCode(m.to);
    const key = `${sender}\u0000${code ?? ''}`;
    if (!out.has(key)) {
      out.set(key, { sender, code, total: 0, blocked: 0, warned: 0, accepted: 0, sids: [] });
    }
    const row = out.get(key);
    row.total += 1;
    const err = errorCode(m);
    if (BLOCKING.includes(err)) {
      row.blocked += 1;
      if (row.sids.length < 3) row.sids.push(m.sid);
    } else if (err === WARNING) {
      row.warned += 1;
    } else {
      row.accepted += 1;
    }
  }
  return out;
}

/**
 * Classify one sender-and-country pair. `configured` is the set of alpha_sender
 * strings attached to the account's Messaging Services, or null when they were
 * not read; it is not a registration list, because no API returns one.
 * Pure. Returns [state, detail].
 */
export function verdict(row, configured = null) {
  const sender = String(row.sender ?? '');
  const code = row.code ?? null;
  let where = code ? `+${code}` : 'an unresolved destination';
  if (code && Object.prototype.hasOwnProperty.call(REGISTRATION_REQUIRED, code)) {
    where = REGISTRATION_REQUIRED[code];
  }
  const total = Number(row.total ?? 0);
  const blocked = Number(row.blocked ?? 0);
  const warned = Number(row.warned ?? 0);

  const known = configured ? [...configured] : [];
  const exact = known.includes(sender);
  const near = !exact && known.some((s) => s.toLowerCase() === sender.toLowerCase());

  if (blocked) {
    if (near) {
      return ['case-mismatch',
        `${blocked} of ${total} to ${where} rejected with 30040/30041, and ` +
        `'${sender}' differs from a configured sender only in case. Sender IDs ` +
        'are matched byte for byte, so this is a change in your sending code, ' +
        'not a registration.'];
    }
    return ['unregistered',
      `${blocked} of ${total} to ${where} rejected with 30040/30041. The ` +
      'destination carrier requires this sender to be pre-registered there; the ' +
      'API accepted every one of these because it cannot know that.'];
  }

  if (warned) {
    return ['warned',
      `${warned} of ${total} to ${where} carry 30018. That is the ` +
      'warning-level sibling of 30041 and it is below the error threshold most ' +
      'alert sweeps use, so this is the notice you would otherwise miss.'];
  }

  if (configured !== null && !exact) {
    return ['not-in-pool',
      `${total} message(s) to ${where} delivering from '${sender}', which is ` +
      'not attached to any Messaging Service. It works today, but nothing on ' +
      'the account records that this string is a sender of yours.'];
  }

  return ['delivering', `${total} message(s) to ${where}, none rejected`];
}

function authHeader(key, secret) {
  return `Basic ${Buffer.from(`${key}:${secret}`).toString('base64')}`;
}

async function get(auth, url, params = {}) {
  const u = new URL(url);
  for (const [k, v] of Object.entries(params)) u.searchParams.set(k, v);
  const res = await fetch(u, { headers: { Authorization: auth } });
  if (res.status === 401 || res.status === 403) {
    throw new Error(`${res.status} from Twilio: check TWILIO_ACCOUNT_SID and ` +
                    'that the API key belongs to that account with read access');
  }
  if (!res.ok) throw new Error(`${res.status} from ${u.pathname}`);
  return res.json();
}

export async function listMessages(auth, account, since, limit = 20000) {
  let url = `${BASE}/Accounts/${account}/Messages.json`;
  let params = { PageSize: 1000, 'DateSent>=': since };
  const out = [];
  while (url && out.length < limit) {
    const page = await get(auth, url, params);
    out.push(...(page.messages ?? []));
    url = page.next_page_uri ? HOST + page.next_page_uri : null;
    params = {};
  }
  return out.slice(0, limit);
}

async function configuredSenders(auth) {
  const owners = new Map();
  const services = (await get(auth, `${MESSAGING}/Services`, { PageSize: 100 })).services ?? [];
  for (const svc of services) {
    const page = await get(auth, `${MESSAGING}/Services/${svc.sid}/AlphaSenders`,
                           { PageSize: 100 });
    for (const alpha of page.alpha_senders ?? []) {
      owners.set(String(alpha.alpha_sender ?? ''), svc.sid);
    }
  }
  return owners;
}

async function main() {
  const account = process.env.TWILIO_ACCOUNT_SID;
  const key = process.env.TWILIO_API_KEY;
  const secret = process.env.TWILIO_API_SECRET;
  if (!account || !key || !secret) {
    console.error('set TWILIO_ACCOUNT_SID, TWILIO_API_KEY and TWILIO_API_SECRET ' +
                  '(an API Key with read access, not the auth token)');
    process.exitCode = 2;
    return;
  }
  const auth = authHeader(key, secret);
  const days = Number(process.argv[process.argv.indexOf('--days') + 1]) || 7;
  const skipServices = process.argv.includes('--skip-services');

  const since = new Date(Date.now() - days * 86400000).toISOString().slice(0, 10);
  const pairs = tally(await listMessages(auth, account, since));
  if (pairs.size === 0) {
    console.log(`no messages from an alphanumeric sender since ${since}`);
    return;
  }

  const owners = skipServices ? new Map() : await configuredSenders(auth);
  const configured = skipServices ? null : new Set(owners.keys());

  let bad = 0;
  for (const key2 of [...pairs.keys()].sort()) {
    const row = pairs.get(key2);
    const [state, detail] = verdict(row, configured);
    const line = `${state.padEnd(14)} ${row.sender.slice(0, 12).padEnd(12)} ${detail}`;
    if (state === 'delivering') { console.log(line); continue; }
    bad += 1;
    console.warn(line);
    if (row.sids.length) console.warn(`  message sids: ${row.sids.join(', ')}`);
    if (state === 'case-mismatch') {
      const match = [...owners.keys()].find(
        (s) => s.toLowerCase() === row.sender.toLowerCase());
      console.warn(`  repair: send From='${match}', the string already ` +
                   `configured on service ${owners.get(match)}. No registration ` +
                   'is needed for that.');
    } else if (state === 'unregistered') {
      console.warn(`  repair: register '${row.sender}' for this country at ` +
                   'Console -> Messaging -> Senders -> Alphanumeric Sender IDs, ' +
                   `then attach it with a create call on ${MESSAGING}/Services/` +
                   '{ServiceSid}/AlphaSenders. Until it is approved, route this ' +
                   'country through a long code.');
    } else {
      console.warn(`  repair: attach '${row.sender}' to the Messaging Service ` +
                   'that should own it, so the account records it as a sender.');
    }
  }

  console.log(`${pairs.size} sender/destination pair(s), ${bad} rejected by the ` +
              'destination carrier');
  process.exitCode = bad ? 1 : 0;
}

// Only run when invoked directly, so importing this module from the test file
// does not start an audit and fail on the missing credentials.
if (import.meta.url === `file://${process.argv[1]}`) {
  main().catch((err) => { console.error(err.message); process.exitCode = 2; });
}

from twilio_alpha_sender_audit import dial_code, sender_kind, tally, verdict


def make(sender, to, code=None, direction="outbound-api", sid="SM1"):
    return {"from": sender, "to": to, "error_code": code, "direction": direction,
            "sid": sid}


def test_sender_blocked_in_one_country_is_unregistered_there():
    rows = tally([make("MyBrand", "+919812345678", 30041),
                  make("MyBrand", "+919812345679", 30040)])
    state, detail = verdict(rows[("MyBrand", "91")], {"MyBrand"})
    assert state == "unregistered"
    assert "India" in detail


def test_the_same_sender_is_healthy_in_the_next_country():
    # Grouping by sender alone would average this into the Indian failures and
    # report one mostly working sender.
    rows = tally([make("MyBrand", "+919812345678", 30041),
                  make("MyBrand", "+33612345678")])
    assert verdict(rows[("MyBrand", "33")], {"MyBrand"})[0] == "delivering"


def test_case_difference_is_reported_as_a_code_change_not_a_registration():
    rows = tally([make("MYBRAND", "+919812345678", 30041)])
    state, detail = verdict(rows[("MYBRAND", "91")], {"MyBrand"})
    assert state == "case-mismatch"
    assert "byte for byte" in detail


def test_30018_is_reported_before_anything_is_blocked():
    rows = tally([make("MyBrand", "+9715012345678", 30018)])
    state, detail = verdict(rows[("MyBrand", "971")], {"MyBrand"})
    assert state == "warned"
    assert "30018" in detail


def test_working_sender_missing_from_every_service_is_its_own_state():
    rows = tally([make("Ghost", "+33612345678")])
    assert verdict(rows[("Ghost", "33")], {"MyBrand"})[0] == "not-in-pool"
    # With the services unread there is nothing to compare against, so no claim.
    assert verdict(rows[("Ghost", "33")], None)[0] == "delivering"


def test_only_alphanumeric_senders_are_counted():
    assert tally([make("+15005550006", "+33612345678"),
                  make("12345", "+33612345678")]) == {}
    assert sender_kind("MyBrand") == "alphanumeric"
    assert sender_kind("12345") == "short-code"


def test_dial_code_prefers_the_longest_match():
    assert dial_code("+971501234567") == "971"
    assert dial_code("+919812345678") == "91"
    assert dial_code("07700900123") is None

import { test } from 'node:test';
import assert from 'node:assert/strict';
import { dialCode, senderKind, tally, verdict } from './twilio-alpha-sender-audit.mjs';

const make = (sender, to, code = null, direction = 'outbound-api') => ({
  from: sender, to, error_code: code, direction, sid: 'SM1',
});
const row = (rows, sender, code) => rows.get(`${sender}\u0000${code}`);

test('sender blocked in one country is unregistered there', () => {
  const rows = tally([make('MyBrand', '+919812345678', 30041),
                      make('MyBrand', '+919812345679', 30040)]);
  const [state, detail] = verdict(row(rows, 'MyBrand', '91'), new Set(['MyBrand']));
  assert.equal(state, 'unregistered');
  assert.match(detail, /India/);
});

test('the same sender is healthy in the next country', () => {
  const rows = tally([make('MyBrand', '+919812345678', 30041),
                      make('MyBrand', '+33612345678')]);
  assert.equal(verdict(row(rows, 'MyBrand', '33'), new Set(['MyBrand']))[0], 'delivering');
});

test('case difference is a code change, not a registration', () => {
  const rows = tally([make('MYBRAND', '+919812345678', 30041)]);
  const [state, detail] = verdict(row(rows, 'MYBRAND', '91'), new Set(['MyBrand']));
  assert.equal(state, 'case-mismatch');
  assert.match(detail, /byte for byte/);
});

test('30018 is reported before anything is blocked', () => {
  const rows = tally([make('MyBrand', '+9715012345678', 30018)]);
  const [state, detail] = verdict(row(rows, 'MyBrand', '971'), new Set(['MyBrand']));
  assert.equal(state, 'warned');
  assert.match(detail, /30018/);
});

test('working sender missing from every service is its own state', () => {
  const rows = tally([make('Ghost', '+33612345678')]);
  assert.equal(verdict(row(rows, 'Ghost', '33'), new Set(['MyBrand']))[0], 'not-in-pool');
  assert.equal(verdict(row(rows, 'Ghost', '33'), null)[0], 'delivering');
});

test('only alphanumeric senders are counted', () => {
  assert.equal(tally([make('+15005550006', '+33612345678'),
                      make('12345', '+33612345678')]).size, 0);
  assert.equal(senderKind('MyBrand'), 'alphanumeric');
  assert.equal(senderKind('12345'), 'short-code');
});

test('dialCode prefers the longest match', () => {
  assert.equal(dialCode('+971501234567'), '971');
  assert.equal(dialCode('+919812345678'), '91');
  assert.equal(dialCode('07700900123'), null);
});