a 10DLC number outside the sender pool is never registered

"""Find SMS-capable US long codes that sit outside any registered A2P sender pool.

Read only. GET requests and nothing else: give this an API Key with read access
rather than the account auth token. The repair is printed, never performed,
because this script holds a credential to an account that can send messages and
spend money.
"""
import argparse
import logging
import os
import sys
from datetime import datetime, timedelta, timezone

import requests

logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s")
log = logging.getLogger("twilio_10dlc_sender_pool_gap")

HOST = "https://api.twilio.com"
BASE = HOST + "/2010-04-01"
MSG = "https://messaging.twilio.com/v1"

UNREGISTERED = "30034"
# Toll-free has its own verification path and its own failure code, 30032. A
# report that mixes the two sends people to the wrong console page.
TOLLFREE_NPA = {"800", "833", "844", "855", "866", "877", "888"}


def is_us_long_code(phone_number):
    """True for a +1 ten digit number that is not toll-free. Pure.

    Short codes, non-US numbers and toll-free numbers are all out of scope for
    10DLC, and each of them has a different registration story.
    """
    number = str(phone_number or "")
    if not number.startswith("+1") or len(number) != 12 or not number[1:].isdigit():
        return False
    return number[2:5] not in TOLLFREE_NPA


def sms_capable(number):
    return bool((number.get("capabilities") or {}).get("sms"))


def bare_from_share(failures):
    """The share of these failures sent with a From rather than a service SID.

    A bare From bypasses the Messaging Service, and therefore the campaign the
    service carries, which is how a gap survives a green compliance dashboard.
    """
    if not failures:
        return 0.0
    bare = len([m for m in failures if not m.get("messaging_service_sid")])
    return bare / float(len(failures))


def verdict(number, service, failures):
    """Classify one owned number. Pure.

    number is an IncomingPhoneNumbers row, service is the Messaging Service
    whose pool contains it or None, failures are that number's 30034 rows.
    Returns (state, detail).
    """
    phone = str(number.get("phone_number") or "")
    if not sms_capable(number):
        return ("not-in-scope", "capabilities.sms is false: not an SMS sender.")
    if not is_us_long_code(phone):
        return ("not-in-scope",
                "not a US long code, so 10DLC registration does not govern it. "
                "Toll-free numbers verify separately and fail with 30032.")

    if service is None:
        if failures:
            return ("sending-direct",
                    "%d x 30034 from a number that is in no Messaging Service "
                    "pool, %.0f%% of them sent with a bare From. A2P approval "
                    "attaches through the pool, so this number is UNREGISTERED "
                    "whatever the brand and campaign say."
                    % (len(failures), bare_from_share(failures) * 100))
        return ("outside-the-pool",
                "SMS capable US long code in no Messaging Service pool, with no "
                "traffic yet. The first US A2P send from it will 30034.")

    name = service.get("friendly_name") or service.get("sid") or "?"
    if not service.get("us_app_to_person_registered"):
        return ("pool-without-a-campaign",
                "in the pool of %s, which has no A2P campaign at all. The pool "
                "is not the problem here; the service is." % name)

    if failures:
        return ("registered-but-failing",
                "%d x 30034 from a number that is already in %s. Either it was "
                "added in the last two weeks and is still PENDING_REGISTRATION, "
                "or the brand is Sole Proprietor and this is the extra number "
                "that never registers." % (len(failures), name))

    return ("registered", "in the pool of %s, which has a campaign." % name)


def get(session, url, **params):
    r = session.get(url, params=params, timeout=30)
    if r.status_code in (401, 403):
        raise SystemExit("%d from Twilio: check TWILIO_ACCOUNT_SID and that the "
                         "API key belongs to that account with read access"
                         % r.status_code)
    r.raise_for_status()
    return r.json()


def list_2010(session, url, key, limit=2000, **params):
    """Page a 2010-04-01 list. next_page_uri is a path, not a URL."""
    out = []
    params = dict(params, PageSize=1000)
    while url and len(out) < limit:
        page = get(session, url, **params)
        out.extend(page.get(key, []))
        nxt = page.get("next_page_uri")
        url, params = (HOST + nxt) if nxt else None, {}
    return out[:limit]


def list_v1(session, url, key, limit=1000):
    """Page a messaging.twilio.com list. meta.next_page_url is absolute."""
    out = []
    while url and len(out) < limit:
        page = get(session, url, PageSize=50)
        out.extend(page.get(key, []))
        url = (page.get("meta") or {}).get("next_page_url")
    return out[:limit]


def main():
    ap = argparse.ArgumentParser(description=__doc__)
    ap.add_argument("--days", type=int, default=7,
                    help="how far back to read the Messages list for 30034s")
    ap.add_argument("--max-messages", type=int, default=20000,
                    help="stop paging the Messages list after this many rows")
    args = ap.parse_args()

    account = os.environ.get("TWILIO_ACCOUNT_SID")
    key = os.environ.get("TWILIO_API_KEY")
    secret = os.environ.get("TWILIO_API_SECRET")
    if not (account and key and secret):
        log.error("set TWILIO_ACCOUNT_SID, TWILIO_API_KEY and TWILIO_API_SECRET "
                  "(an API Key with read access, not the auth token)")
        return 2

    session = requests.Session()
    session.auth = (key, secret)

    numbers = list_2010(session, "%s/Accounts/%s/IncomingPhoneNumbers.json"
                        % (BASE, account), "incoming_phone_numbers")

    pool = {}
    for service in list_v1(session, MSG + "/Services", "services"):
        for entry in list_v1(session, "%s/Services/%s/PhoneNumbers"
                             % (MSG, service["sid"]), "phone_numbers"):
            pool[str(entry.get("phone_number"))] = service

    since = (datetime.now(timezone.utc)
             - timedelta(days=args.days)).strftime("%Y-%m-%d")
    failures = {}
    for message in list_2010(session, "%s/Accounts/%s/Messages.json" % (BASE, account),
                             "messages", args.max_messages,
                             **{"DateSent>=": since}):
        if str(message.get("error_code") or "") == UNREGISTERED:
            failures.setdefault(str(message.get("from") or ""), []).append(message)

    in_scope = bad = 0
    for number in numbers:
        phone = str(number.get("phone_number") or "")
        state, detail = verdict(number, pool.get(phone), failures.get(phone, []))
        if state == "not-in-scope":
            continue
        in_scope += 1
        line = "%-23s %s  %s" % (state, phone, detail)
        if state == "registered":
            log.info(line)
            continue
        bad += 1
        log.warning(line)
        if state in ("sending-direct", "outside-the-pool"):
            log.warning("  repair: POST %s/Services/{ServiceSid}/PhoneNumbers with "
                        "PhoneNumberSid=%s, then send with MessagingServiceSid "
                        "rather than a bare From", MSG, number.get("sid", "PN..."))
        elif state == "pool-without-a-campaign":
            log.warning("  repair: register a campaign on that Messaging Service "
                        "before touching the pool")
        else:
            log.warning("  repair: wait out the carrier registration window "
                        "before changing anything; removing and re-adding the "
                        "number restarts it")

    log.info("%d US long code(s), %d outside a registered sender pool",
             in_scope, bad)
    return 1 if bad else 0


if __name__ == "__main__":
    sys.exit(main())

/**
 * Find SMS-capable US long codes that sit outside any registered A2P sender pool.
 *
 * Read only. GET requests and nothing else: give this an API Key with read
 * access rather than the account auth token. The repair is printed, never
 * performed.
 */
const HOST = 'https://api.twilio.com';
const BASE = `${HOST}/2010-04-01`;
const MSG = 'https://messaging.twilio.com/v1';

const UNREGISTERED = '30034';
// Toll-free has its own verification path and its own failure code, 30032.
const TOLLFREE_NPA = new Set(['800', '833', '844', '855', '866', '877', '888']);

/**
 * True for a +1 ten digit number that is not toll-free. Pure. Short codes,
 * non-US numbers and toll-free numbers are all out of scope for 10DLC.
 */
export function isUsLongCode(phoneNumber) {
  const number = String(phoneNumber ?? '');
  if (!number.startsWith('+1') || number.length !== 12) return false;
  if (!/^[0-9]+$/.test(number.slice(1))) return false;
  return !TOLLFREE_NPA.has(number.slice(2, 5));
}

export function smsCapable(number) {
  return Boolean(number.capabilities?.sms);
}

/**
 * The share of these failures sent with a From rather than a service SID. A
 * bare From bypasses the Messaging Service, and therefore the campaign it
 * carries, which is how a gap survives a green compliance dashboard.
 */
export function bareFromShare(failures) {
  if (failures.length === 0) return 0;
  return failures.filter((m) => !m.messaging_service_sid).length / failures.length;
}

/**
 * Classify one owned number. Pure. number is an IncomingPhoneNumbers row,
 * service is the Messaging Service whose pool contains it or null, failures are
 * that number's 30034 rows. Returns [state, detail].
 */
export function verdict(number, service, failures) {
  const phone = String(number.phone_number ?? '');
  if (!smsCapable(number)) {
    return ['not-in-scope', 'capabilities.sms is false: not an SMS sender.'];
  }
  if (!isUsLongCode(phone)) {
    return ['not-in-scope',
      'not a US long code, so 10DLC registration does not govern it. Toll-free ' +
      'numbers verify separately and fail with 30032.'];
  }

  if (!service) {
    if (failures.length) {
      return ['sending-direct',
        `${failures.length} x 30034 from a number that is in no Messaging ` +
        `Service pool, ${(bareFromShare(failures) * 100).toFixed(0)}% of them ` +
        'sent with a bare From. A2P approval attaches through the pool, so this ' +
        'number is UNREGISTERED whatever the brand and campaign say.'];
    }
    return ['outside-the-pool',
      'SMS capable US long code in no Messaging Service pool, with no traffic ' +
      'yet. The first US A2P send from it will 30034.'];
  }

  const name = service.friendly_name ?? service.sid ?? '?';
  if (!service.us_app_to_person_registered) {
    return ['pool-without-a-campaign',
      `in the pool of ${name}, which has no A2P campaign at all. The pool is ` +
      'not the problem here; the service is.'];
  }

  if (failures.length) {
    return ['registered-but-failing',
      `${failures.length} x 30034 from a number that is already in ${name}. ` +
      'Either it was added in the last two weeks and is still ' +
      'PENDING_REGISTRATION, or the brand is Sole Proprietor and this is the ' +
      'extra number that never registers.'];
  }

  return ['registered', `in the pool of ${name}, which has a campaign.`];
}

function authHeader(key, secret) {
  return `Basic ${Buffer.from(`${key}:${secret}`).toString('base64')}`;
}

async function get(auth, url, params = {}) {
  const u = new URL(url);
  for (const [k, v] of Object.entries(params)) u.searchParams.set(k, v);
  const res = await fetch(u, { headers: { Authorization: auth } });
  if (res.status === 401 || res.status === 403) {
    throw new Error(`${res.status} from Twilio: check TWILIO_ACCOUNT_SID and ` +
                    'that the API key belongs to that account with read access');
  }
  if (!res.ok) throw new Error(`${res.status} from ${u.pathname}`);
  return res.json();
}

async function list2010(auth, url, key, limit = 2000, extra = {}) {
  const out = [];
  let next = url;
  let params = { ...extra, PageSize: 1000 };
  while (next && out.length < limit) {
    const page = await get(auth, next, params);
    out.push(...(page[key] ?? []));
    next = page.next_page_uri ? HOST + page.next_page_uri : null;
    params = {};
  }
  return out.slice(0, limit);
}

async function listV1(auth, url, key, limit = 1000) {
  const out = [];
  let next = url;
  while (next && out.length < limit) {
    const page = await get(auth, next, { PageSize: 50 });
    out.push(...(page[key] ?? []));
    next = page.meta?.next_page_url ?? null;
  }
  return out.slice(0, limit);
}

async function main() {
  const account = process.env.TWILIO_ACCOUNT_SID;
  const key = process.env.TWILIO_API_KEY;
  const secret = process.env.TWILIO_API_SECRET;
  if (!account || !key || !secret) {
    console.error('set TWILIO_ACCOUNT_SID, TWILIO_API_KEY and TWILIO_API_SECRET ' +
                  '(an API Key with read access, not the auth token)');
    process.exitCode = 2;
    return;
  }
  const auth = authHeader(key, secret);

  const numbers = await list2010(auth,
    `${BASE}/Accounts/${account}/IncomingPhoneNumbers.json`, 'incoming_phone_numbers');

  const pool = new Map();
  for (const service of await listV1(auth, `${MSG}/Services`, 'services')) {
    for (const entry of await listV1(auth,
      `${MSG}/Services/${service.sid}/PhoneNumbers`, 'phone_numbers')) {
      pool.set(String(entry.phone_number), service);
    }
  }

  const days = Number(process.argv.includes('--days')
    ? process.argv[process.argv.indexOf('--days') + 1] : 7) || 7;
  const since = new Date(Date.now() - days * 86400000).toISOString().slice(0, 10);

  const failures = new Map();
  const messages = await list2010(auth, `${BASE}/Accounts/${account}/Messages.json`,
    'messages', 20000, { 'DateSent>=': since });
  for (const message of messages) {
    if (String(message.error_code ?? '') !== UNREGISTERED) continue;
    const from = String(message.from ?? '');
    failures.set(from, [...(failures.get(from) ?? []), message]);
  }

  let inScope = 0;
  let bad = 0;
  for (const number of numbers) {
    const phone = String(number.phone_number ?? '');
    const [state, detail] = verdict(number, pool.get(phone) ?? null,
                                    failures.get(phone) ?? []);
    if (state === 'not-in-scope') continue;
    inScope += 1;
    const line = `${state.padEnd(23)} ${phone}  ${detail}`;
    if (state === 'registered') { console.log(line); continue; }
    bad += 1;
    console.warn(line);
    if (state === 'sending-direct' || state === 'outside-the-pool') {
      console.warn(`  repair: POST ${MSG}/Services/{ServiceSid}/PhoneNumbers ` +
                   `with PhoneNumberSid=${number.sid ?? 'PN...'}, then send with ` +
                   'MessagingServiceSid rather than a bare From');
    } else if (state === 'pool-without-a-campaign') {
      console.warn('  repair: register a campaign on that Messaging Service ' +
                   'before touching the pool');
    } else {
      console.warn('  repair: wait out the carrier registration window before ' +
                   'changing anything; removing and re-adding the number ' +
                   'restarts it');
    }
  }

  console.log(`${inScope} US long code(s), ${bad} outside a registered sender pool`);
  process.exitCode = bad ? 1 : 0;
}

// Only run when invoked directly. The test file imports this module, and without
// the guard main() would run there too, fail on the missing credentials and set a
// non-zero exit code that fails the whole test file even as every test passes.
if (import.meta.url === `file://${process.argv[1]}`) {
  main().catch((err) => { console.error(err.message); process.exitCode = 2; });
}

from twilio_10dlc_sender_pool_gap import (bare_from_share, is_us_long_code,
                                             verdict)

LONG_CODE = {"sid": "PN1", "phone_number": "+15125550123",
             "capabilities": {"sms": True, "voice": True}}
REGISTERED = {"sid": "MG1", "friendly_name": "prod",
              "us_app_to_person_registered": True}
UNREGISTERED_SERVICE = {"sid": "MG2", "friendly_name": "staging",
                        "us_app_to_person_registered": False}


def fail(**kw):
    return dict({"error_code": 30034, "from": "+15125550123"}, **kw)


def test_a_number_in_no_pool_that_is_failing_is_sending_direct():
    state, detail = verdict(LONG_CODE, None, [fail(), fail()])
    assert state == "sending-direct"
    assert "100%" in detail and "UNREGISTERED" in detail


def test_a_number_in_no_pool_with_no_traffic_is_latent_not_broken():
    state, detail = verdict(LONG_CODE, None, [])
    assert state == "outside-the-pool"
    assert "will 30034" in detail


def test_a_pool_on_a_service_with_no_campaign_points_at_the_service():
    state, detail = verdict(LONG_CODE, UNREGISTERED_SERVICE, [fail()])
    assert state == "pool-without-a-campaign"
    assert "staging" in detail


def test_a_pooled_number_that_still_fails_may_just_be_new():
    # The one finding here where the right action is to wait rather than change
    # anything, so it must not share a state with the gaps.
    state, detail = verdict(LONG_CODE, REGISTERED, [fail()])
    assert state == "registered-but-failing"
    assert "PENDING_REGISTRATION" in detail


def test_a_pooled_number_with_no_failures_is_clean():
    assert verdict(LONG_CODE, REGISTERED, [])[0] == "registered"


def test_toll_free_is_out_of_scope_and_says_why():
    tf = dict(LONG_CODE, phone_number="+18885550123")
    state, detail = verdict(tf, None, [])
    assert state == "not-in-scope"
    assert "30032" in detail


def test_a_number_that_cannot_send_sms_is_out_of_scope():
    voice_only = dict(LONG_CODE, capabilities={"sms": False, "voice": True})
    assert verdict(voice_only, None, [fail()])[0] == "not-in-scope"


def test_scope_is_us_ten_digit_long_codes_only():
    assert is_us_long_code("+15125550123")
    assert not is_us_long_code("+442071838750")
    assert not is_us_long_code("+18445550123")
    assert not is_us_long_code("12345")
    assert not is_us_long_code(None)


def test_a_send_carrying_a_service_sid_is_not_a_bare_from():
    assert bare_from_share([fail(messaging_service_sid="MG1"), fail()]) == 0.5

import { test } from 'node:test';
import assert from 'node:assert/strict';
import { bareFromShare, isUsLongCode, verdict }
  from './twilio-10dlc-sender-pool-gap.mjs';

const LONG_CODE = { sid: 'PN1', phone_number: '+15125550123',
  capabilities: { sms: true, voice: true } };
const REGISTERED = { sid: 'MG1', friendly_name: 'prod',
  us_app_to_person_registered: true };
const UNREGISTERED_SERVICE = { sid: 'MG2', friendly_name: 'staging',
  us_app_to_person_registered: false };

const fail = (extra = {}) => ({ error_code: 30034, from: '+15125550123', ...extra });

test('a number in no pool that is failing is sending direct', () => {
  const [state, detail] = verdict(LONG_CODE, null, [fail(), fail()]);
  assert.equal(state, 'sending-direct');
  assert.match(detail, /100%/);
  assert.match(detail, /UNREGISTERED/);
});

test('a number in no pool with no traffic is latent, not broken', () => {
  const [state, detail] = verdict(LONG_CODE, null, []);
  assert.equal(state, 'outside-the-pool');
  assert.match(detail, /will 30034/);
});

test('a pool on a service with no campaign points at the service', () => {
  const [state, detail] = verdict(LONG_CODE, UNREGISTERED_SERVICE, [fail()]);
  assert.equal(state, 'pool-without-a-campaign');
  assert.match(detail, /staging/);
});

test('a pooled number that still fails may just be new', () => {
  const [state, detail] = verdict(LONG_CODE, REGISTERED, [fail()]);
  assert.equal(state, 'registered-but-failing');
  assert.match(detail, /PENDING_REGISTRATION/);
});

test('a pooled number with no failures is clean', () => {
  assert.equal(verdict(LONG_CODE, REGISTERED, [])[0], 'registered');
});

test('toll free is out of scope and says why', () => {
  const [state, detail] = verdict({ ...LONG_CODE, phone_number: '+18885550123' },
                                  null, []);
  assert.equal(state, 'not-in-scope');
  assert.match(detail, /30032/);
});

test('a number that cannot send sms is out of scope', () => {
  const voiceOnly = { ...LONG_CODE, capabilities: { sms: false, voice: true } };
  assert.equal(verdict(voiceOnly, null, [fail()])[0], 'not-in-scope');
});

test('scope is us ten digit long codes only', () => {
  assert.equal(isUsLongCode('+15125550123'), true);
  assert.equal(isUsLongCode('+442071838750'), false);
  assert.equal(isUsLongCode('+18445550123'), false);
  assert.equal(isUsLongCode('12345'), false);
  assert.equal(isUsLongCode(null), false);
});

test('a send carrying a service sid is not a bare from', () => {
  assert.equal(bareFromShare([fail({ messaging_service_sid: 'MG1' }), fail()]), 0.5);
});