one smart quote triples your segment count and your bill

"""Report Twilio messages inflated into UCS-2 by a handful of characters.

Read only. GET requests and nothing else: give this an API Key with read access
rather than the account auth token. The repair is printed, never performed,
because this script holds a credential to an account that can send messages and
spend money.
"""
import argparse
import datetime as dt
import logging
import os
import sys

import requests

logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s")
log = logging.getLogger("twilio_segment_audit")

HOST = "https://api.twilio.com"
BASE = HOST + "/2010-04-01"
MESSAGING = "https://messaging.twilio.com/v1"

# GSM 03.38, the alphabet a single segment of 160 characters is drawn from.
GSM_BASIC = set(
    "@£$¥èéùìòÇØøÅåΔ_ΦΓΛΩΠΨΣΘΞÆæßÉ !#¤%&()*+,-./0123456789:;<=>?"
    "¡ABCDEFGHIJKLMNOPQRSTUVWXYZÄÖÑܧ¿abcdefghijklmnopqrstuvwxyzäöñüà")
# The four that cannot sit in the literal above without fighting the quoting:
# double quote, apostrophe, newline, carriage return.
GSM_BASIC.update({chr(34), chr(39), chr(10), chr(13)})

# The extension table. These are GSM-7, but each one costs two units, which is
# the detail that makes a naive len() check wrong by a whole segment.
GSM_EXT = set("^{}[~]|€")
GSM_EXT.add(chr(92))  # backslash

GSM_SINGLE, GSM_MULTI = 160, 153
UCS_SINGLE, UCS_MULTI = 70, 67

# What Smart Encoding substitutes, near enough: the characters a rich text
# editor inserts silently and that nobody meant to pay three times for.
TRANSLITERATE = {
    "‘": chr(39), "’": chr(39), "‚": chr(39), "‛": chr(39),
    "′": chr(39), "´": chr(39), "ʼ": chr(39),
    "“": chr(34), "”": chr(34), "„": chr(34),
    "«": chr(34), "»": chr(34),
    "–": "-", "—": "-", "−": "-",
    "…": "...", " ": " ", "•": "*", "™": "TM",
}


def sms_encoding(body):
    """GSM-7 if every character is in the GSM alphabet, UCS-2 otherwise. Pure.

    The choice is per message, not per character: one character outside the
    alphabet moves the entire body to UCS-2 and 70 characters a segment.
    """
    for c in str(body or ""):
        if c not in GSM_BASIC and c not in GSM_EXT:
            return "UCS-2"
    return "GSM-7"


def segments(body):
    """Return (encoding, units, segment_count) for a body. Pure.

    Units, not characters: an extension character costs two in GSM-7, and a
    character outside the Basic Multilingual Plane (every emoji) costs two
    UTF-16 code units in UCS-2.
    """
    text = str(body or "")
    encoding = sms_encoding(text)
    if encoding == "GSM-7":
        units = sum(2 if c in GSM_EXT else 1 for c in text)
        single, multi = GSM_SINGLE, GSM_MULTI
    else:
        units = sum(2 if ord(c) > 0xFFFF else 1 for c in text)
        single, multi = UCS_SINGLE, UCS_MULTI
    if units <= single:
        return (encoding, units, 1)
    return (encoding, units, -(-units // multi))


def offenders(body):
    """Every distinct character forcing UCS-2, with its substitute or None.

    Pure. None means nothing can stand in for it: an emoji, or a script that is
    simply not Latin, in which case UCS-2 is correct and the cost is real.
    """
    out, seen = [], set()
    for c in str(body or ""):
        if c in GSM_BASIC or c in GSM_EXT or c in seen:
            continue
        seen.add(c)
        out.append((c, TRANSLITERATE.get(c)))
    return out


def transliterate(body):
    """The body as Smart Encoding would rewrite it. Pure."""
    return "".join(TRANSLITERATE.get(c, c) for c in str(body or ""))


def describe(chars):
    return ", ".join("%s (U+%04X)" % (c, ord(c)) for c in chars)


def verdict(body, reported=None):
    """Classify one message body. Pure, and the whole point of this script.

    `reported` is num_segments as Twilio billed it. When it is lower than the
    raw body would cost, Smart Encoding rewrote the message on the way out: the
    template is still wrong, a setting is just paying for it.

    Returns (state, detail).
    """
    text = str(body or "")
    encoding, units, count = segments(text)
    if encoding == "GSM-7":
        return ("gsm-7", "%d segment(s), GSM-7, %d unit(s)" % (count, units))

    if reported is not None:
        try:
            billed = int(reported)
        except (TypeError, ValueError):
            billed = None
        if billed is not None and billed < count:
            return ("smart-encoded",
                    "billed %d segment(s), not the %d this body costs as UCS-2: "
                    "Smart Encoding rewrote it on the way out, so the template "
                    "is still wrong and a setting is paying for it."
                    % (billed, count))

    found = offenders(text)
    fixable = [c for c, sub in found if sub is not None]
    stuck = [c for c, sub in found if sub is None]

    if stuck:
        return ("ucs2-required",
                "%d segment(s) as UCS-2, %d unit(s). Nothing to strip: %s cannot "
                "be transliterated, so UCS-2 is correct here and the cost is "
                "expected rather than accidental."
                % (count, units, describe(stuck[:4])))

    clean = segments(transliterate(text))[2]
    return ("ucs2-avoidable",
            "%d segment(s) as UCS-2 against %d after transliteration: %d extra "
            "segment(s) on every send of this body, caused by %s."
            % (count, clean, count - clean, describe(fixable[:4])))


def tally(messages):
    """Bucket outbound messages by sender and add up the avoidable segments.

    Pure. Inbound messages are skipped: their encoding is the sender's problem
    and you are not billed by the segment for receiving them.
    """
    rows = {}
    for m in messages:
        if str(m.get("direction") or "").startswith("inbound"):
            continue
        body = str(m.get("body") or "")
        if not body.strip():
            continue
        key = m.get("messaging_service_sid") or m.get("from") or "unknown sender"
        row = rows.setdefault(key, {"total": 0, "ucs2": 0, "extra": 0,
                                    "chars": [], "sids": []})
        row["total"] += 1
        state, _ = verdict(body, m.get("num_segments"))
        if state == "gsm-7":
            continue
        row["ucs2"] += 1
        if state == "ucs2-avoidable":
            row["extra"] += segments(body)[2] - segments(transliterate(body))[2]
        for c, _sub in offenders(body):
            if c not in row["chars"]:
                row["chars"].append(c)
        if len(row["sids"]) < 3:
            row["sids"].append(m.get("sid"))
    return rows


def get(session, url, **params):
    r = session.get(url, params=params, timeout=30)
    if r.status_code in (401, 403):
        raise SystemExit("%d from Twilio: check TWILIO_ACCOUNT_SID and that the "
                         "API key belongs to that account with read access"
                         % r.status_code)
    r.raise_for_status()
    return r.json()


def list_messages(session, account, since, limit):
    """Page Messages.json. Nothing to filter on: this failure has no error
    code, so the window and the cap are the only bounds there are."""
    url = "%s/Accounts/%s/Messages.json" % (BASE, account)
    params = {"PageSize": 1000, "DateSent>=": since}
    out = []
    while url and len(out) < limit:
        page = get(session, url, **params)
        out.extend(page.get("messages", []))
        nxt = page.get("next_page_uri")
        url, params = (HOST + nxt) if nxt else None, {}
    return out[:limit]


def smart_encoding_by_service(session):
    """Map service sid to its smart_encoding flag. next_page_url is absolute on
    this API, unlike the relative next_page_uri on the 2010 one."""
    url = "%s/Services" % MESSAGING
    params = {"PageSize": 50}
    out = {}
    while url:
        page = get(session, url, **params)
        for s in page.get("services", []):
            out[s.get("sid")] = bool(s.get("smart_encoding"))
        url = (page.get("meta") or {}).get("next_page_url")
        params = {}
    return out


def main():
    ap = argparse.ArgumentParser(description=__doc__)
    ap.add_argument("--days", type=int, default=7,
                    help="how far back to read the Messages list")
    ap.add_argument("--max-messages", type=int, default=20000,
                    help="stop paging after this many messages")
    args = ap.parse_args()

    account = os.environ.get("TWILIO_ACCOUNT_SID")
    key = os.environ.get("TWILIO_API_KEY")
    secret = os.environ.get("TWILIO_API_SECRET")
    if not (account and key and secret):
        log.error("set TWILIO_ACCOUNT_SID, TWILIO_API_KEY and TWILIO_API_SECRET "
                  "(an API Key with read access, not the auth token)")
        return 2

    session = requests.Session()
    session.auth = (key, secret)

    since = (dt.date.today() - dt.timedelta(days=args.days)).isoformat()
    messages = list_messages(session, account, since, args.max_messages)
    if not messages:
        log.info("no messages sent since %s", since)
        return 0

    senders = tally(messages)
    services = smart_encoding_by_service(session)

    extra = 0
    for sender, stats in sorted(senders.items()):
        if not stats["ucs2"]:
            log.info("%-15s %s  %d message(s), all GSM-7",
                     "gsm-7", sender, stats["total"])
            continue
        extra += stats["extra"]
        state = "inflated" if stats["extra"] else "ucs2"
        log.warning("%-15s %s  %d of %d message(s) in UCS-2, %d extra "
                    "segment(s) over the window, offenders: %s",
                    state, sender, stats["ucs2"], stats["total"], stats["extra"],
                    describe(stats["chars"][:6]))
        log.warning("  message sids: %s", ", ".join(str(s) for s in stats["sids"]))
        if str(sender).startswith("MG"):
            if services.get(sender):
                log.warning("  smart_encoding is already true on %s: what is "
                            "left is genuinely non-GSM content, or a template "
                            "using characters the substitution table misses.",
                            sender)
            else:
                log.warning("  repair: POST %s/Services/%s SmartEncoding=true, "
                            "and normalise curly quotes and dashes where the "
                            "template is authored.", MESSAGING, sender)
        else:
            log.warning("  repair: this sent with a bare From, so no Messaging "
                        "Service and no Smart Encoding to enable. Send through "
                        "a service, or normalise the body before the call.")

    log.info("%d sender(s) over %d day(s), %d extra segment(s) from avoidable "
             "UCS-2", len(senders), args.days, extra)
    return 1 if extra else 0


if __name__ == "__main__":
    sys.exit(main())

/**
 * Report Twilio messages inflated into UCS-2 by a handful of characters.
 *
 * Read only. GET requests and nothing else: give this an API Key with read
 * access rather than the account auth token. The repair is printed, never
 * performed.
 */
const HOST = 'https://api.twilio.com';
const BASE = `${HOST}/2010-04-01`;
const MESSAGING = 'https://messaging.twilio.com/v1';

// GSM 03.38, the alphabet a single segment of 160 characters is drawn from.
const GSM_BASIC = new Set(
  '@£$¥èéùìòÇØøÅåΔ_ΦΓΛΩΠΨΣΘΞÆæßÉ !#¤%&()*+,-./0123456789:;<=>?' +
  '¡ABCDEFGHIJKLMNOPQRSTUVWXYZÄÖÑܧ¿abcdefghijklmnopqrstuvwxyzäöñüà');
// Double quote, apostrophe, newline and carriage return, kept out of the
// literal above so it does not fight the quoting.
for (const code of [34, 39, 10, 13]) GSM_BASIC.add(String.fromCharCode(code));

// The extension table: GSM-7, but two units each.
const GSM_EXT = new Set('^{}[~]|€');
GSM_EXT.add(String.fromCharCode(92)); // backslash

const GSM_SINGLE = 160, GSM_MULTI = 153;
const UCS_SINGLE = 70, UCS_MULTI = 67;

// What Smart Encoding substitutes, near enough.
const TRANSLITERATE = new Map(Object.entries({
  '‘': "'", '’': "'", '‚': "'", '‛': "'",
  '′': "'", '´': "'", 'ʼ': "'",
  '“': '"', '”': '"', '„': '"', '«': '"', '»': '"',
  '–': '-', '—': '-', '−': '-',
  '…': '...', ' ': ' ', '•': '*', '™': 'TM',
}));

/**
 * GSM-7 if every character is in the GSM alphabet, UCS-2 otherwise. Pure. The
 * choice is per message: one character outside the alphabet moves the whole
 * body to 70 characters a segment.
 */
export function smsEncoding(body) {
  for (const c of String(body ?? '')) {
    if (!GSM_BASIC.has(c) && !GSM_EXT.has(c)) return 'UCS-2';
  }
  return 'GSM-7';
}

/**
 * Return [encoding, units, segmentCount]. Pure. Units, not characters: an
 * extension character costs two in GSM-7, and anything outside the Basic
 * Multilingual Plane costs two UTF-16 code units in UCS-2.
 */
export function segments(body) {
  const text = String(body ?? '');
  const encoding = smsEncoding(text);
  let units = 0;
  for (const c of text) {
    if (encoding === 'GSM-7') units += GSM_EXT.has(c) ? 2 : 1;
    else units += c.codePointAt(0) > 0xFFFF ? 2 : 1;
  }
  const single = encoding === 'GSM-7' ? GSM_SINGLE : UCS_SINGLE;
  const multi = encoding === 'GSM-7' ? GSM_MULTI : UCS_MULTI;
  return [encoding, units, units <= single ? 1 : Math.ceil(units / multi)];
}

/**
 * Every distinct character forcing UCS-2, with its substitute or null. Pure.
 * null means nothing can stand in for it, and UCS-2 is correct.
 */
export function offenders(body) {
  const out = [];
  const seen = new Set();
  for (const c of String(body ?? '')) {
    if (GSM_BASIC.has(c) || GSM_EXT.has(c) || seen.has(c)) continue;
    seen.add(c);
    out.push([c, TRANSLITERATE.get(c) ?? null]);
  }
  return out;
}

/** The body as Smart Encoding would rewrite it. Pure. */
export function transliterate(body) {
  return [...String(body ?? '')].map((c) => TRANSLITERATE.get(c) ?? c).join('');
}

export function describe(chars) {
  return chars.map((c) =>
    `${c} (U+${c.codePointAt(0).toString(16).toUpperCase().padStart(4, '0')})`)
    .join(', ');
}

/**
 * Classify one message body. Pure, and the whole point of this script.
 * `reported` is num_segments as Twilio billed it; lower than the raw cost means
 * Smart Encoding rewrote the message on the way out. Returns [state, detail].
 */
export function verdict(body, reported = null) {
  const text = String(body ?? '');
  const [encoding, units, count] = segments(text);
  if (encoding === 'GSM-7') {
    return ['gsm-7', `${count} segment(s), GSM-7, ${units} unit(s)`];
  }

  if (reported !== null && reported !== undefined) {
    const billed = Number(reported);
    if (Number.isFinite(billed) && billed < count) {
      return ['smart-encoded',
        `billed ${billed} segment(s), not the ${count} this body costs as ` +
        'UCS-2: Smart Encoding rewrote it on the way out, so the template is ' +
        'still wrong and a setting is paying for it.'];
    }
  }

  const found = offenders(text);
  const fixable = found.filter(([, sub]) => sub !== null).map(([c]) => c);
  const stuck = found.filter(([, sub]) => sub === null).map(([c]) => c);

  if (stuck.length) {
    return ['ucs2-required',
      `${count} segment(s) as UCS-2, ${units} unit(s). Nothing to strip: ` +
      `${describe(stuck.slice(0, 4))} cannot be transliterated, so UCS-2 is ` +
      'correct here and the cost is expected rather than accidental.'];
  }

  const clean = segments(transliterate(text))[2];
  return ['ucs2-avoidable',
    `${count} segment(s) as UCS-2 against ${clean} after transliteration: ` +
    `${count - clean} extra segment(s) on every send of this body, caused by ` +
    `${describe(fixable.slice(0, 4))}.`];
}

/**
 * Bucket outbound messages by sender and add up the avoidable segments. Pure.
 */
export function tally(messages) {
  const rows = new Map();
  for (const m of messages) {
    if (String(m.direction ?? '').startsWith('inbound')) continue;
    const body = String(m.body ?? '');
    if (!body.trim()) continue;
    const key = m.messaging_service_sid || m.from || 'unknown sender';
    if (!rows.has(key)) rows.set(key, { total: 0, ucs2: 0, extra: 0, chars: [], sids: [] });
    const row = rows.get(key);
    row.total += 1;
    const [state] = verdict(body, m.num_segments ?? null);
    if (state === 'gsm-7') continue;
    row.ucs2 += 1;
    if (state === 'ucs2-avoidable') {
      row.extra += segments(body)[2] - segments(transliterate(body))[2];
    }
    for (const [c] of offenders(body)) if (!row.chars.includes(c)) row.chars.push(c);
    if (row.sids.length < 3) row.sids.push(m.sid);
  }
  return rows;
}

function authHeader(key, secret) {
  return `Basic ${Buffer.from(`${key}:${secret}`).toString('base64')}`;
}

async function get(auth, url, params = {}) {
  const u = new URL(url);
  for (const [k, v] of Object.entries(params)) u.searchParams.set(k, v);
  const res = await fetch(u, { headers: { Authorization: auth } });
  if (res.status === 401 || res.status === 403) {
    throw new Error(`${res.status} from Twilio: check TWILIO_ACCOUNT_SID and ` +
                    'that the API key belongs to that account with read access');
  }
  if (!res.ok) throw new Error(`${res.status} from ${u.pathname}`);
  return res.json();
}

async function listMessages(auth, account, since, limit) {
  let url = `${BASE}/Accounts/${account}/Messages.json`;
  let params = { PageSize: 1000, 'DateSent>=': since };
  const out = [];
  while (url && out.length < limit) {
    const page = await get(auth, url, params);
    out.push(...(page.messages ?? []));
    url = page.next_page_uri ? HOST + page.next_page_uri : null;
    params = {};
  }
  return out.slice(0, limit);
}

async function smartEncodingByService(auth) {
  let url = `${MESSAGING}/Services`;
  let params = { PageSize: 50 };
  const out = new Map();
  while (url) {
    const page = await get(auth, url, params);
    for (const s of page.services ?? []) out.set(s.sid, Boolean(s.smart_encoding));
    url = page.meta?.next_page_url ?? null;
    params = {};
  }
  return out;
}

function flag(name, fallback) {
  const i = process.argv.indexOf(name);
  return i === -1 ? fallback : Number(process.argv[i + 1]);
}

async function main() {
  const account = process.env.TWILIO_ACCOUNT_SID;
  const key = process.env.TWILIO_API_KEY;
  const secret = process.env.TWILIO_API_SECRET;
  if (!account || !key || !secret) {
    console.error('set TWILIO_ACCOUNT_SID, TWILIO_API_KEY and TWILIO_API_SECRET ' +
                  '(an API Key with read access, not the auth token)');
    process.exitCode = 2;
    return;
  }
  const auth = authHeader(key, secret);
  const days = flag('--days', 7);
  const since = new Date(Date.now() - days * 86400000).toISOString().slice(0, 10);

  const messages = await listMessages(auth, account, since, flag('--max-messages', 20000));
  if (messages.length === 0) {
    console.log(`no messages sent since ${since}`);
    return;
  }

  const senders = tally(messages);
  const services = await smartEncodingByService(auth);

  let extra = 0;
  for (const sender of [...senders.keys()].sort()) {
    const stats = senders.get(sender);
    if (!stats.ucs2) {
      console.log(`gsm-7           ${sender}  ${stats.total} message(s), all GSM-7`);
      continue;
    }
    extra += stats.extra;
    const state = stats.extra ? 'inflated' : 'ucs2';
    console.warn(`${state.padEnd(15)} ${sender}  ${stats.ucs2} of ${stats.total} ` +
                 `message(s) in UCS-2, ${stats.extra} extra segment(s) over the ` +
                 `window, offenders: ${describe(stats.chars.slice(0, 6))}`);
    console.warn(`  message sids: ${stats.sids.join(', ')}`);
    if (String(sender).startsWith('MG')) {
      if (services.get(sender)) {
        console.warn(`  smart_encoding is already true on ${sender}: what is left ` +
                     'is genuinely non-GSM content, or characters the ' +
                     'substitution table misses.');
      } else {
        console.warn(`  repair: POST ${MESSAGING}/Services/${sender} ` +
                     'SmartEncoding=true, and normalise curly quotes and dashes ' +
                     'where the template is authored.');
      }
    } else {
      console.warn('  repair: this sent with a bare From, so no Messaging Service ' +
                   'and no Smart Encoding to enable. Send through a service, or ' +
                   'normalise the body before the call.');
    }
  }

  console.log(`${senders.size} sender(s) over ${days} day(s), ${extra} extra ` +
              'segment(s) from avoidable UCS-2');
  process.exitCode = extra ? 1 : 0;
}

// Only run when invoked directly, so importing this module in the tests does not
// run main(), fail on the missing credentials and set a non-zero exit code.
if (import.meta.url === `file://${process.argv[1]}`) {
  main().catch((err) => { console.error(err.message); process.exitCode = 2; });
}

from twilio_segment_audit import (offenders, segments, sms_encoding, tally,
                                  transliterate, verdict)

CURLY = "’"     # right single quotation mark, the usual culprit
PARTY = "🎉"  # an emoji, outside the Basic Multilingual Plane


def test_plain_ascii_is_gsm7():
    assert sms_encoding("Your code is 123456") == "GSM-7"


def test_one_curly_apostrophe_moves_the_whole_body_to_ucs2():
    assert sms_encoding("It%ss ready" % CURLY) == "UCS-2"


def test_gsm7_segment_boundary_is_160_then_153():
    assert segments("a" * 160) == ("GSM-7", 160, 1)
    assert segments("a" * 161)[2] == 2
    assert segments("a" * 306)[2] == 2
    assert segments("a" * 307)[2] == 3


def test_extension_characters_cost_two_units():
    # 80 euro signs is 160 units: still one segment, but at half the characters.
    assert segments("€" * 80) == ("GSM-7", 160, 1)
    assert segments("€" * 81)[2] == 2


def test_ucs2_segment_boundary_is_70_then_67():
    body = "а" * 70  # Cyrillic
    assert segments(body) == ("UCS-2", 70, 1)
    assert segments("а" * 71)[2] == 2


def test_an_emoji_costs_two_utf16_units():
    encoding, units, count = segments(PARTY * 40)
    assert encoding == "UCS-2"
    assert units == 80
    assert count == 2


def test_one_smart_quote_turns_one_segment_into_three():
    body = "a" * 149 + CURLY
    state, detail = verdict(body)
    assert state == "ucs2-avoidable"
    assert segments(body)[2] == 3
    assert segments(transliterate(body))[2] == 1
    assert "2 extra segment(s)" in detail


def test_an_emoji_is_ucs2_that_nothing_can_fix():
    state, detail = verdict("Sale today " + PARTY)
    assert state == "ucs2-required"
    assert "cannot be transliterated" in detail


def test_billing_fewer_segments_means_smart_encoding_already_ran():
    state, detail = verdict("a" * 149 + CURLY, reported=1)
    assert state == "smart-encoded"
    assert "still wrong" in detail


def test_offenders_are_deduplicated_and_carry_their_substitute():
    found = offenders("%s%s ok %s" % (CURLY, CURLY, PARTY))
    assert [c for c, _ in found] == [CURLY, PARTY]
    assert found[0][1] == chr(39)
    assert found[1][1] is None


def test_tally_adds_up_the_avoidable_segments_per_sender():
    body = "a" * 149 + CURLY
    rows = tally([
        {"sid": "SM1", "messaging_service_sid": "MG1", "body": body},
        {"sid": "SM2", "messaging_service_sid": "MG1", "body": body},
        {"sid": "SM3", "messaging_service_sid": "MG1", "body": "plain text"},
        {"sid": "SM4", "from": "+15550001111", "direction": "inbound", "body": body},
    ])
    assert list(rows) == ["MG1"]
    assert rows["MG1"] == {"total": 3, "ucs2": 2, "extra": 4,
                           "chars": [CURLY], "sids": ["SM1", "SM2"]}

import { test } from 'node:test';
import assert from 'node:assert/strict';
import { offenders, segments, smsEncoding, tally, transliterate, verdict }
  from './twilio-segment-audit.mjs';

const CURLY = '’';       // right single quotation mark, the usual culprit
const PARTY = '🎉'; // an emoji, outside the Basic Multilingual Plane

test('plain ascii is gsm-7', () => {
  assert.equal(smsEncoding('Your code is 123456'), 'GSM-7');
});

test('one curly apostrophe moves the whole body to ucs-2', () => {
  assert.equal(smsEncoding(`It${CURLY}s ready`), 'UCS-2');
});

test('gsm-7 segment boundary is 160 then 153', () => {
  assert.deepEqual(segments('a'.repeat(160)), ['GSM-7', 160, 1]);
  assert.equal(segments('a'.repeat(161))[2], 2);
  assert.equal(segments('a'.repeat(306))[2], 2);
  assert.equal(segments('a'.repeat(307))[2], 3);
});

test('extension characters cost two units', () => {
  assert.deepEqual(segments('€'.repeat(80)), ['GSM-7', 160, 1]);
  assert.equal(segments('€'.repeat(81))[2], 2);
});

test('ucs-2 segment boundary is 70 then 67', () => {
  assert.deepEqual(segments('а'.repeat(70)), ['UCS-2', 70, 1]);
  assert.equal(segments('а'.repeat(71))[2], 2);
});

test('an emoji costs two utf-16 units', () => {
  assert.deepEqual(segments(PARTY.repeat(40)), ['UCS-2', 80, 2]);
});

test('one smart quote turns one segment into three', () => {
  const body = 'a'.repeat(149) + CURLY;
  const [state, detail] = verdict(body);
  assert.equal(state, 'ucs2-avoidable');
  assert.equal(segments(body)[2], 3);
  assert.equal(segments(transliterate(body))[2], 1);
  assert.match(detail, /2 extra segment\(s\)/);
});

test('an emoji is ucs-2 that nothing can fix', () => {
  const [state, detail] = verdict(`Sale today ${PARTY}`);
  assert.equal(state, 'ucs2-required');
  assert.match(detail, /cannot be transliterated/);
});

test('billing fewer segments means smart encoding already ran', () => {
  const [state, detail] = verdict('a'.repeat(149) + CURLY, 1);
  assert.equal(state, 'smart-encoded');
  assert.match(detail, /still wrong/);
});

test('offenders are deduplicated and carry their substitute', () => {
  const found = offenders(`${CURLY}${CURLY} ok ${PARTY}`);
  assert.deepEqual(found.map(([c]) => c), [CURLY, PARTY]);
  assert.equal(found[0][1], "'");
  assert.equal(found[1][1], null);
});

test('tally adds up the avoidable segments per sender', () => {
  const body = 'a'.repeat(149) + CURLY;
  const rows = tally([
    { sid: 'SM1', messaging_service_sid: 'MG1', body },
    { sid: 'SM2', messaging_service_sid: 'MG1', body },
    { sid: 'SM3', messaging_service_sid: 'MG1', body: 'plain text' },
    { sid: 'SM4', from: '+15550001111', direction: 'inbound', body },
  ]);
  assert.deepEqual([...rows.keys()], ['MG1']);
  assert.deepEqual(rows.get('MG1'), { total: 3, ucs2: 2, extra: 4,
                                      chars: [CURLY], sids: ['SM1', 'SM2'] });
});