an SMS number that belongs to no Messaging Service

"""Report SMS-capable Twilio numbers that are in no Messaging Service.

Read only. GET requests and nothing else: give this an API Key with read access
rather than the account auth token. The repair is printed, never performed,
because this script holds a credential to an account that can send messages and
spend money.
"""
import argparse
import datetime as dt
import logging
import os
import sys

import requests

logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s")
log = logging.getLogger("twilio_unpooled_number_audit")

HOST = "https://api.twilio.com"
BASE = HOST + "/2010-04-01"
MESSAGING = "https://messaging.twilio.com/v1"

# What a number outside every service does not get. All four are implemented by
# sender selection, which only runs when a send names a MessagingServiceSid.
LOST = ("no sticky sender, no geomatch, no long code failover, and the A2P "
        "campaign attaches through a pool this number is not in")


def verdict(number, service=None, traffic=None):
    """Classify one owned number against the Messaging Service holding it.

    `number` is an IncomingPhoneNumber. `service` is the Messaging Service whose
    pool contains it, or None when no pool does. `traffic` is how many outbound
    messages were seen from it in the window, or None when traffic was not
    checked at all: not checked and none found are different facts, and merging
    them makes an idle number look like an urgent one.

    Pure, so the scope rule and the priority rule can be tested without a
    network. Returns (state, detail).
    """
    caps = number.get("capabilities") or {}
    if not caps.get("sms"):
        return ("out-of-scope",
                "capabilities.sms is false, so a sender pool has nothing to "
                "offer it. Voice only numbers are somebody else's report.")

    if service:
        label = service.get("friendly_name") or service.get("sid") or "a service"
        return ("pooled", "in the sender pool of %s" % label)

    if traffic is None:
        return ("unpooled",
                "SMS capable and in no Messaging Service: %s." % LOST)
    if traffic > 0:
        return ("unpooled-sending",
                "sending today with no Messaging Service behind it, at least "
                "%d message(s) in the window: %s." % (traffic, LOST))
    return ("unpooled-idle",
            "SMS capable, in no Messaging Service, and nothing sent in the "
            "window. Pool it before somebody uses it, or release it.")


def get(session, url, **params):
    r = session.get(url, params=params, timeout=30)
    if r.status_code in (401, 403):
        raise SystemExit("%d from Twilio: check TWILIO_ACCOUNT_SID and that the "
                         "API key belongs to that account with read access"
                         % r.status_code)
    r.raise_for_status()
    return r.json()


def list_numbers(session, account, limit):
    """Page IncomingPhoneNumbers. next_page_uri is a path, not an absolute URL."""
    url = "%s/Accounts/%s/IncomingPhoneNumbers.json" % (BASE, account)
    params = {"PageSize": 100}
    out = []
    while url and len(out) < limit:
        page = get(session, url, **params)
        out.extend(page.get("incoming_phone_numbers", []))
        nxt = page.get("next_page_uri")
        url, params = (HOST + nxt) if nxt else None, {}
    return out[:limit]


def list_services(session, limit):
    """Page Messaging Services. This API pages on meta.next_page_url, which is
    absolute, unlike the 2010-04-01 API next door."""
    url = "%s/Services" % MESSAGING
    params = {"PageSize": 100}
    out = []
    while url and len(out) < limit:
        page = get(session, url, **params)
        out.extend(page.get("services", []))
        url = (page.get("meta") or {}).get("next_page_url")
        params = {}
    return out[:limit]


def pooled_by_sid(session, services):
    """Map every pooled PN sid to the service holding it.

    Keyed on the sid rather than the E.164 string: both sides return the sid,
    and a sid does not lose to a formatting difference.
    """
    owner = {}
    for svc in services:
        url = "%s/Services/%s/PhoneNumbers" % (MESSAGING, svc.get("sid"))
        params = {"PageSize": 100}
        while url:
            page = get(session, url, **params)
            for entry in page.get("phone_numbers", []):
                owner[entry.get("sid")] = svc
            url = (page.get("meta") or {}).get("next_page_url")
            params = {}
    return owner


def outbound_count(session, account, e164, days):
    """One row is enough to know the number is in use. PageSize=1 keeps this to
    a single small response per flagged number."""
    since = (dt.date.today() - dt.timedelta(days=days)).isoformat()
    page = get(session, "%s/Accounts/%s/Messages.json" % (BASE, account),
               **{"From": e164, "DateSent>": since, "PageSize": 1})
    return len(page.get("messages", []))


def main():
    ap = argparse.ArgumentParser(description=__doc__)
    ap.add_argument("--max-numbers", type=int, default=1000,
                    help="stop paging after this many numbers")
    ap.add_argument("--max-services", type=int, default=200,
                    help="stop paging after this many Messaging Services")
    ap.add_argument("--check-traffic", action="store_true",
                    help="one extra GET per unpooled number to see if it sends")
    ap.add_argument("--days", type=int, default=90,
                    help="traffic window in days for --check-traffic")
    args = ap.parse_args()

    account = os.environ.get("TWILIO_ACCOUNT_SID")
    key = os.environ.get("TWILIO_API_KEY")
    secret = os.environ.get("TWILIO_API_SECRET")
    if not (account and key and secret):
        log.error("set TWILIO_ACCOUNT_SID, TWILIO_API_KEY and TWILIO_API_SECRET "
                  "(an API Key with read access, not the auth token)")
        return 2

    session = requests.Session()
    session.auth = (key, secret)

    numbers = list_numbers(session, account, args.max_numbers)
    services = list_services(session, args.max_services)
    owner = pooled_by_sid(session, services)
    log.info("%d number(s) on the account, %d Messaging Service(s), %d pooled sender(s)",
             len(numbers), len(services), len(owner))

    considered = bad = 0
    for n in numbers:
        service = owner.get(n.get("sid"))
        traffic = None
        if service is None and args.check_traffic and (n.get("capabilities") or {}).get("sms"):
            traffic = outbound_count(session, account, n.get("phone_number"), args.days)
        state, detail = verdict(n, service, traffic)
        if state == "out-of-scope":
            continue
        considered += 1
        line = "%-16s %s  %s" % (state, n.get("phone_number", "?"), detail)
        if state == "pooled":
            log.info(line)
            continue
        bad += 1
        log.warning(line)
        log.warning("  repair: POST %s/Services/{ServiceSid}/PhoneNumbers "
                    "PhoneNumberSid=%s, then send with MessagingServiceSid "
                    "instead of a bare From so sender selection actually runs.",
                    MESSAGING, n.get("sid"))

    log.info("%d SMS capable number(s), %d outside every Messaging Service",
             considered, bad)
    return 1 if bad else 0


if __name__ == "__main__":
    sys.exit(main())

/**
 * Report SMS-capable Twilio numbers that are in no Messaging Service.
 *
 * Read only. GET requests and nothing else: give this an API Key with read
 * access rather than the account auth token. The repair is printed, never
 * performed.
 */
const HOST = 'https://api.twilio.com';
const BASE = `${HOST}/2010-04-01`;
const MESSAGING = 'https://messaging.twilio.com/v1';

// What a number outside every service does not get. All four are implemented by
// sender selection, which only runs when a send names a MessagingServiceSid.
const LOST = 'no sticky sender, no geomatch, no long code failover, and the A2P ' +
             'campaign attaches through a pool this number is not in';

/**
 * Classify one owned number against the Messaging Service holding it. `service`
 * is the service whose pool contains it or null; `traffic` is how many outbound
 * messages were seen in the window, or null when traffic was not checked. Not
 * checked and none found are different facts. Pure. Returns [state, detail].
 */
export function verdict(number, service = null, traffic = null) {
  const caps = number.capabilities ?? {};
  if (!caps.sms) {
    return ['out-of-scope',
      'capabilities.sms is false, so a sender pool has nothing to offer it. ' +
      "Voice only numbers are somebody else's report."];
  }

  if (service) {
    const label = service.friendly_name ?? service.sid ?? 'a service';
    return ['pooled', `in the sender pool of ${label}`];
  }

  if (traffic === null || traffic === undefined) {
    return ['unpooled', `SMS capable and in no Messaging Service: ${LOST}.`];
  }
  if (traffic > 0) {
    return ['unpooled-sending',
      'sending today with no Messaging Service behind it, at least ' +
      `${traffic} message(s) in the window: ${LOST}.`];
  }
  return ['unpooled-idle',
    'SMS capable, in no Messaging Service, and nothing sent in the window. ' +
    'Pool it before somebody uses it, or release it.'];
}

function authHeader(key, secret) {
  return `Basic ${Buffer.from(`${key}:${secret}`).toString('base64')}`;
}

async function get(auth, url, params = {}) {
  const u = new URL(url);
  for (const [k, v] of Object.entries(params)) u.searchParams.set(k, v);
  const res = await fetch(u, { headers: { Authorization: auth } });
  if (res.status === 401 || res.status === 403) {
    throw new Error(`${res.status} from Twilio: check TWILIO_ACCOUNT_SID and ` +
                    'that the API key belongs to that account with read access');
  }
  if (!res.ok) throw new Error(`${res.status} from ${u.pathname}`);
  return res.json();
}

export async function listNumbers(auth, account, limit = 1000) {
  let url = `${BASE}/Accounts/${account}/IncomingPhoneNumbers.json`;
  let params = { PageSize: 100 };
  const out = [];
  while (url && out.length < limit) {
    const page = await get(auth, url, params);
    out.push(...(page.incoming_phone_numbers ?? []));
    url = page.next_page_uri ? HOST + page.next_page_uri : null;
    params = {};
  }
  return out.slice(0, limit);
}

export async function listServices(auth, limit = 200) {
  let url = `${MESSAGING}/Services`;
  let params = { PageSize: 100 };
  const out = [];
  while (url && out.length < limit) {
    const page = await get(auth, url, params);
    out.push(...(page.services ?? []));
    url = page.meta?.next_page_url ?? null;
    params = {};
  }
  return out.slice(0, limit);
}

/** Map every pooled PN sid to the service holding it. Keyed on the sid. */
async function pooledBySid(auth, services) {
  const owner = new Map();
  for (const svc of services) {
    let url = `${MESSAGING}/Services/${svc.sid}/PhoneNumbers`;
    let params = { PageSize: 100 };
    while (url) {
      const page = await get(auth, url, params);
      for (const entry of page.phone_numbers ?? []) owner.set(entry.sid, svc);
      url = page.meta?.next_page_url ?? null;
      params = {};
    }
  }
  return owner;
}

async function outboundCount(auth, account, e164, days) {
  const since = new Date(Date.now() - days * 86400000).toISOString().slice(0, 10);
  const page = await get(auth, `${BASE}/Accounts/${account}/Messages.json`,
                         { From: e164, 'DateSent>': since, PageSize: 1 });
  return (page.messages ?? []).length;
}

function flagValue(name, fallback) {
  const i = process.argv.indexOf(name);
  return i === -1 ? fallback : Number(process.argv[i + 1]);
}

async function main() {
  const account = process.env.TWILIO_ACCOUNT_SID;
  const key = process.env.TWILIO_API_KEY;
  const secret = process.env.TWILIO_API_SECRET;
  if (!account || !key || !secret) {
    console.error('set TWILIO_ACCOUNT_SID, TWILIO_API_KEY and TWILIO_API_SECRET ' +
                  '(an API Key with read access, not the auth token)');
    process.exitCode = 2;
    return;
  }
  const auth = authHeader(key, secret);
  const checkTraffic = process.argv.includes('--check-traffic');
  const days = flagValue('--days', 90);

  const numbers = await listNumbers(auth, account);
  const services = await listServices(auth);
  const owner = await pooledBySid(auth, services);
  console.log(`${numbers.length} number(s) on the account, ${services.length} ` +
              `Messaging Service(s), ${owner.size} pooled sender(s)`);

  let considered = 0;
  let bad = 0;
  for (const n of numbers) {
    const service = owner.get(n.sid) ?? null;
    let traffic = null;
    if (!service && checkTraffic && (n.capabilities ?? {}).sms) {
      traffic = await outboundCount(auth, account, n.phone_number, days);
    }
    const [state, detail] = verdict(n, service, traffic);
    if (state === 'out-of-scope') continue;
    considered += 1;
    const line = `${state.padEnd(16)} ${n.phone_number ?? '?'}  ${detail}`;
    if (state === 'pooled') { console.log(line); continue; }
    bad += 1;
    console.warn(line);
    console.warn(`  repair: POST ${MESSAGING}/Services/{ServiceSid}/PhoneNumbers ` +
                 `PhoneNumberSid=${n.sid}, then send with MessagingServiceSid ` +
                 'instead of a bare From so sender selection actually runs.');
  }

  console.log(`${considered} SMS capable number(s), ${bad} outside every Messaging Service`);
  process.exitCode = bad ? 1 : 0;
}

// Only run when invoked directly. The test file imports this module, and without
// the guard main() would run there too, fail on the missing credentials and set a
// non-zero exit code that fails the whole test file even as every test passes.
if (import.meta.url === `file://${process.argv[1]}`) {
  main().catch((err) => { console.error(err.message); process.exitCode = 2; });
}

from twilio_unpooled_number_audit import verdict

SMS = {"phone_number": "+15550001111", "sid": "PN1", "capabilities": {"sms": True, "voice": True}}
VOICE_ONLY = {"phone_number": "+15550002222", "sid": "PN2", "capabilities": {"sms": False, "voice": True}}
SERVICE = {"sid": "MG1", "friendly_name": "transactional"}


def test_a_number_in_a_pool_is_not_a_finding():
    state, detail = verdict(SMS, SERVICE)
    assert state == "pooled"
    assert "transactional" in detail


def test_an_unpooled_number_names_what_it_is_missing():
    state, detail = verdict(SMS)
    assert state == "unpooled"
    assert "sticky sender" in detail
    assert "geomatch" in detail


def test_a_voice_only_number_is_out_of_scope():
    # A sender pool cannot help it, and listing it trains people to skim.
    state, detail = verdict(VOICE_ONLY)
    assert state == "out-of-scope"
    assert "capabilities.sms is false" in detail


def test_unchecked_traffic_is_not_the_same_as_no_traffic():
    assert verdict(SMS, None, None)[0] == "unpooled"
    assert verdict(SMS, None, 0)[0] == "unpooled-idle"


def test_an_unpooled_number_that_is_sending_is_the_urgent_one():
    state, detail = verdict(SMS, None, 4)
    assert state == "unpooled-sending"
    assert "at least 4 message(s)" in detail


def test_pool_membership_beats_traffic():
    # Being in a pool settles it; the traffic count is only there to rank the
    # numbers that are not.
    assert verdict(SMS, SERVICE, 500)[0] == "pooled"


def test_a_service_with_no_friendly_name_falls_back_to_its_sid():
    _state, detail = verdict(SMS, {"sid": "MG9"})
    assert "MG9" in detail


def test_missing_capabilities_object_is_treated_as_not_sms():
    assert verdict({"phone_number": "+15550003333", "sid": "PN3"})[0] == "out-of-scope"

import { test } from 'node:test';
import assert from 'node:assert/strict';
import { verdict } from './twilio-unpooled-number-audit.mjs';

const SMS = { phone_number: '+15550001111', sid: 'PN1', capabilities: { sms: true, voice: true } };
const VOICE_ONLY = { phone_number: '+15550002222', sid: 'PN2', capabilities: { sms: false, voice: true } };
const SERVICE = { sid: 'MG1', friendly_name: 'transactional' };

test('a number in a pool is not a finding', () => {
  const [state, detail] = verdict(SMS, SERVICE);
  assert.equal(state, 'pooled');
  assert.match(detail, /transactional/);
});

test('an unpooled number names what it is missing', () => {
  const [state, detail] = verdict(SMS);
  assert.equal(state, 'unpooled');
  assert.match(detail, /sticky sender/);
  assert.match(detail, /geomatch/);
});

test('a voice only number is out of scope', () => {
  const [state, detail] = verdict(VOICE_ONLY);
  assert.equal(state, 'out-of-scope');
  assert.match(detail, /capabilities\.sms is false/);
});

test('unchecked traffic is not the same as no traffic', () => {
  assert.equal(verdict(SMS, null, null)[0], 'unpooled');
  assert.equal(verdict(SMS, null, 0)[0], 'unpooled-idle');
});

test('an unpooled number that is sending is the urgent one', () => {
  const [state, detail] = verdict(SMS, null, 4);
  assert.equal(state, 'unpooled-sending');
  assert.match(detail, /at least 4 message\(s\)/);
});

test('pool membership beats traffic', () => {
  assert.equal(verdict(SMS, SERVICE, 500)[0], 'pooled');
});

test('a service with no friendly name falls back to its sid', () => {
  const [, detail] = verdict(SMS, { sid: 'MG9' });
  assert.match(detail, /MG9/);
});

test('missing capabilities object is treated as not sms', () => {
  assert.equal(verdict({ phone_number: '+15550003333', sid: 'PN3' })[0], 'out-of-scope');
});