a Sync webhook rejected as invalid, or never called at all

"""Report Twilio Sync Services whose webhook is invalid or cannot fire (54051).

Read only. GET requests and nothing else: give this an API Key with read access
rather than the account auth token. The repair is printed, never performed,
because this script holds a credential to an account that can send messages and
spend money.
"""
import argparse
import datetime as dt
import logging
import os
import sys

import requests

logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s")
log = logging.getLogger("twilio_sync_webhook_audit")

SYNC = "https://sync.twilio.com/v1"
MONITOR = "https://monitor.twilio.com/v1"

INVALID_WEBHOOK = 54051


def alert_counts(alerts, code=INVALID_WEBHOOK):
    """Count alerts carrying one error code, keyed by resource_sid. Pure.

    error_code arrives as a number on the Alert resource and as a string in some
    exports, so it is coerced rather than compared raw: that comparison is why a
    sweep reports nothing on an account full of findings.

    Keyed rather than totalled because the same code can be attributed to more
    than one resource, and a count against a resource the caller did not ask
    about is still worth printing rather than dropping.
    """
    counts = {}
    for a in alerts or []:
        raw = a.get("error_code")
        try:
            if raw is None or int(raw) != int(code):
                continue
        except (TypeError, ValueError):
            continue
        sid = str(a.get("resource_sid") or "(unattributed)")
        counts[sid] = counts.get(sid, 0) + 1
    return counts


def verdict(service, rest_writes=False, alerts=0):
    """Classify one Sync Service's webhook. Pure, so the one judgement call in
    this note is visible in one place.

    `rest_writes` is the caller saying their application changes Sync data
    through the REST API. It is an input rather than an assumption because
    webhooks_from_rest_enabled being false is correct on a service only ever
    written to by client SDKs, and an outage on one written to by a server.

    `alerts` is how many 54051 alerts named this service in the window.

    Returns (state, detail).
    """
    url = str(service.get("webhook_url") or "").strip()
    from_rest = service.get("webhooks_from_rest_enabled")
    low = url.lower()

    if not url:
        return ("no-url",
                "webhook_url is empty: no change on this service calls anything, "
                "and an attempt to deliver raises 54051.")

    if low.startswith("http://"):
        return ("insecure",
                "webhook_url is plain http, which is rejected as invalid (54051) "
                "and would have carried document contents in the clear.")

    if not low.startswith("https://"):
        return ("not-absolute",
                "webhook_url is %r, which is not an absolute https URL for "
                "Twilio to resolve and connect to." % url)

    if alerts:
        return ("unreachable",
                "%d alert(s) with 54051 named this service while webhook_url is "
                "a well-formed https URL: Twilio could not reach or complete the "
                "request to %s." % (alerts, url))

    if from_rest is False and rest_writes:
        return ("rest-silent",
                "webhooks_from_rest_enabled is false and your application writes "
                "over REST, so none of those changes calls %s. No error is "
                "raised for this." % url)

    if from_rest is False:
        return ("rest-disabled",
                "webhooks_from_rest_enabled is false. Correct if only client SDKs "
                "change this data; silent for every server-side write if not.")

    return ("ok", "https webhook at %s, REST-driven changes included" % url)


def get(session, url, **params):
    r = session.get(url, params=params, timeout=30)
    if r.status_code in (401, 403):
        raise SystemExit("%d from Twilio: check TWILIO_ACCOUNT_SID and that the "
                         "API key belongs to that account with read access"
                         % r.status_code)
    r.raise_for_status()
    return r.json()


def paged(session, url, key, limit, **first):
    params = dict(first)
    params.setdefault("PageSize", 50)
    out = []
    while url and len(out) < limit:
        page = get(session, url, **params)
        out.extend(page.get(key, []))
        url = (page.get("meta") or {}).get("next_page_url")
        params = {}
    return out[:limit]


def main():
    ap = argparse.ArgumentParser(description=__doc__)
    ap.add_argument("--rest-writes", action="store_true",
                    help="your application changes Sync data through the REST API")
    ap.add_argument("--days", type=int, default=7,
                    help="how far back to sweep the alerts (30 day retention)")
    ap.add_argument("--max-alerts", type=int, default=10000)
    args = ap.parse_args()

    account = os.environ.get("TWILIO_ACCOUNT_SID")
    key = os.environ.get("TWILIO_API_KEY")
    secret = os.environ.get("TWILIO_API_SECRET")
    if not (account and key and secret):
        log.error("set TWILIO_ACCOUNT_SID, TWILIO_API_KEY and TWILIO_API_SECRET "
                  "(an API Key with read access, not the auth token)")
        return 2

    session = requests.Session()
    session.auth = (key, secret)

    services = paged(session, SYNC + "/Services", "services", 200)
    if not services:
        log.info("no Sync Services on this account")
        return 0

    start = (dt.date.today() - dt.timedelta(days=args.days)).isoformat()
    counts = alert_counts(paged(session, MONITOR + "/Alerts", "alerts",
                                args.max_alerts, LogLevel="error", StartDate=start,
                                PageSize=100))

    bad = 0
    for svc in services:
        sid = svc.get("sid")
        state, detail = verdict(svc, args.rest_writes, counts.pop(sid, 0))
        line = "%-14s %s  %s" % (state, svc.get("friendly_name") or sid, detail)
        if state in ("ok", "rest-disabled"):
            log.info(line)
            continue
        bad += 1
        log.warning(line)
        log.warning("  repair: update %s/Services/%s with "
                    "WebhookUrl=https://your-app.example.com/sync and, when your "
                    "writes come from the server, WebhooksFromRestEnabled=true.",
                    SYNC, sid)

    for sid, n in sorted(counts.items()):
        log.info("%d alert(s) with 54051 attributed to %s, which is not a Sync "
                 "Service on this account", n, sid)

    log.info("%d service(s), %d with a webhook that cannot fire", len(services), bad)
    return 1 if bad else 0


if __name__ == "__main__":
    sys.exit(main())

/**
 * Report Twilio Sync Services whose webhook is invalid or cannot fire (54051).
 *
 * Read only. GET requests and nothing else: give this an API Key with read
 * access rather than the account auth token. The repair is printed, never
 * performed.
 */
const SYNC = 'https://sync.twilio.com/v1';
const MONITOR = 'https://monitor.twilio.com/v1';

const INVALID_WEBHOOK = 54051;

/**
 * Count alerts carrying one error code, keyed by resource_sid. Pure.
 *
 * error_code arrives as a number on the Alert resource and as a string in some
 * exports, so it is coerced rather than compared raw. Keyed rather than
 * totalled because a count against a resource the caller did not ask about is
 * still worth printing rather than dropping.
 */
export function alertCounts(alerts, code = INVALID_WEBHOOK) {
  const counts = new Map();
  for (const a of alerts ?? []) {
    const raw = a.error_code;
    if (raw === null || raw === undefined || raw === '') continue;
    const n = Number(raw);
    if (!Number.isFinite(n) || n !== Number(code)) continue;
    const sid = String(a.resource_sid ?? '(unattributed)');
    counts.set(sid, (counts.get(sid) ?? 0) + 1);
  }
  return counts;
}

/**
 * Classify one Sync Service's webhook. Pure.
 *
 * `restWrites` is the caller saying their application changes Sync data through
 * the REST API. It is an input rather than an assumption because
 * webhooks_from_rest_enabled being false is correct on a service only ever
 * written to by client SDKs, and an outage on one written to by a server.
 * `alerts` is how many 54051 alerts named this service. Returns [state, detail].
 */
export function verdict(service, restWrites = false, alerts = 0) {
  const url = String(service.webhook_url ?? '').trim();
  const fromRest = service.webhooks_from_rest_enabled;
  const low = url.toLowerCase();

  if (!url) {
    return ['no-url',
      'webhook_url is empty: no change on this service calls anything, and an ' +
      'attempt to deliver raises 54051.'];
  }

  if (low.startsWith('http://')) {
    return ['insecure',
      'webhook_url is plain http, which is rejected as invalid (54051) and ' +
      'would have carried document contents in the clear.'];
  }

  if (!low.startsWith('https://')) {
    return ['not-absolute',
      `webhook_url is "${url}", which is not an absolute https URL for Twilio ` +
      'to resolve and connect to.'];
  }

  if (alerts) {
    return ['unreachable',
      `${alerts} alert(s) with 54051 named this service while webhook_url is a ` +
      `well-formed https URL: Twilio could not reach or complete the request to ${url}.`];
  }

  if (fromRest === false && restWrites) {
    return ['rest-silent',
      'webhooks_from_rest_enabled is false and your application writes over ' +
      `REST, so none of those changes calls ${url}. No error is raised for this.`];
  }

  if (fromRest === false) {
    return ['rest-disabled',
      'webhooks_from_rest_enabled is false. Correct if only client SDKs change ' +
      'this data; silent for every server-side write if not.'];
  }

  return ['ok', `https webhook at ${url}, REST-driven changes included`];
}

function authHeader(key, secret) {
  return `Basic ${Buffer.from(`${key}:${secret}`).toString('base64')}`;
}

async function get(auth, url, params = {}) {
  const u = new URL(url);
  for (const [k, v] of Object.entries(params)) u.searchParams.set(k, v);
  const res = await fetch(u, { headers: { Authorization: auth } });
  if (res.status === 401 || res.status === 403) {
    throw new Error(`${res.status} from Twilio: check TWILIO_ACCOUNT_SID and ` +
                    'that the API key belongs to that account with read access');
  }
  if (!res.ok) throw new Error(`${res.status} from ${u.pathname}`);
  return res.json();
}

async function paged(auth, url, key, limit, first = {}) {
  let params = { PageSize: 50, ...first };
  const out = [];
  while (url && out.length < limit) {
    const page = await get(auth, url, params);
    out.push(...(page[key] ?? []));
    url = (page.meta ?? {}).next_page_url ?? null;
    params = {};
  }
  return out.slice(0, limit);
}

async function main() {
  const account = process.env.TWILIO_ACCOUNT_SID;
  const key = process.env.TWILIO_API_KEY;
  const secret = process.env.TWILIO_API_SECRET;
  if (!account || !key || !secret) {
    console.error('set TWILIO_ACCOUNT_SID, TWILIO_API_KEY and TWILIO_API_SECRET ' +
                  '(an API Key with read access, not the auth token)');
    process.exitCode = 2;
    return;
  }
  const auth = authHeader(key, secret);
  const restWrites = process.argv.includes('--rest-writes');
  const flag = process.argv.indexOf('--days');
  const days = flag === -1 ? 7 : Number(process.argv[flag + 1] ?? 7);

  const services = await paged(auth, `${SYNC}/Services`, 'services', 200);
  if (!services.length) {
    console.log('no Sync Services on this account');
    return;
  }

  const start = new Date(Date.now() - days * 86400000).toISOString().slice(0, 10);
  const counts = alertCounts(await paged(auth, `${MONITOR}/Alerts`, 'alerts', 10000,
                                         { LogLevel: 'error', StartDate: start,
                                           PageSize: 100 }));

  let bad = 0;
  for (const svc of services) {
    const alerts = counts.get(svc.sid) ?? 0;
    counts.delete(svc.sid);
    const [state, detail] = verdict(svc, restWrites, alerts);
    const line = `${state.padEnd(14)} ${svc.friendly_name || svc.sid}  ${detail}`;
    if (state === 'ok' || state === 'rest-disabled') { console.log(line); continue; }
    bad += 1;
    console.warn(line);
    console.warn(`  repair: update ${SYNC}/Services/${svc.sid} with ` +
                 'WebhookUrl=https://your-app.example.com/sync and, when your ' +
                 'writes come from the server, WebhooksFromRestEnabled=true.');
  }

  for (const [sid, n] of [...counts.entries()].sort()) {
    console.log(`${n} alert(s) with 54051 attributed to ${sid}, which is not a ` +
                'Sync Service on this account');
  }

  console.log(`${services.length} service(s), ${bad} with a webhook that cannot fire`);
  process.exitCode = bad ? 1 : 0;
}

// Only run when invoked directly. The test file imports this module, and without
// the guard main() would run there too, fail on the missing credentials and set a
// non-zero exit code that fails the whole test file even as every test passes.
if (import.meta.url === `file://${process.argv[1]}`) {
  main().catch((err) => { console.error(err.message); process.exitCode = 2; });
}

from twilio_sync_webhook_audit import alert_counts, verdict

URL = "https://app.example.com/sync"


def service(**kw):
    base = {"sid": "IS1", "friendly_name": "live", "webhook_url": URL,
            "webhooks_from_rest_enabled": True}
    base.update(kw)
    return base


def test_rest_writes_decide_whether_the_flag_is_a_fault():
    svc = service(webhooks_from_rest_enabled=False)
    assert verdict(svc, rest_writes=False)[0] == "rest-disabled"
    state, detail = verdict(svc, rest_writes=True)
    assert state == "rest-silent"
    assert "No error is raised" in detail


def test_an_empty_webhook_url_is_the_first_thing_reported():
    state, detail = verdict(service(webhook_url="", webhooks_from_rest_enabled=False))
    assert state == "no-url"
    assert "54051" in detail


def test_plain_http_is_rejected_and_insecure():
    state, detail = verdict(service(webhook_url="http://app.example.com/sync"))
    assert state == "insecure"
    assert "in the clear" in detail


def test_a_url_with_no_scheme_is_not_absolute():
    assert verdict(service(webhook_url="app.example.com/sync"))[0] == "not-absolute"


def test_alerts_against_a_well_formed_url_mean_unreachable():
    state, detail = verdict(service(), alerts=12)
    assert state == "unreachable"
    assert "12 alert(s)" in detail


def test_a_healthy_service_is_ok():
    assert verdict(service(), rest_writes=True)[0] == "ok"


def test_alert_counts_coerce_the_code_and_key_on_the_resource():
    alerts = [{"error_code": "54051", "resource_sid": "IS1"},
              {"error_code": 54051, "resource_sid": "IS1"},
              {"error_code": 54051, "resource_sid": "IS2"},
              {"error_code": 11200, "resource_sid": "IS1"},
              {"error_code": None, "resource_sid": "IS1"}]
    assert alert_counts(alerts) == {"IS1": 2, "IS2": 1}


def test_an_alert_with_no_resource_is_still_counted():
    assert alert_counts([{"error_code": 54051}]) == {"(unattributed)": 1}

import { test } from 'node:test';
import assert from 'node:assert/strict';
import { alertCounts, verdict } from './twilio-sync-webhook-audit.mjs';

const URL_ = 'https://app.example.com/sync';

const service = (kw = {}) => ({
  sid: 'IS1', friendly_name: 'live', webhook_url: URL_,
  webhooks_from_rest_enabled: true, ...kw,
});

test('rest writes decide whether the flag is a fault', () => {
  const svc = service({ webhooks_from_rest_enabled: false });
  assert.equal(verdict(svc, false)[0], 'rest-disabled');
  const [state, detail] = verdict(svc, true);
  assert.equal(state, 'rest-silent');
  assert.match(detail, /No error is raised/);
});

test('an empty webhook url is the first thing reported', () => {
  const [state, detail] = verdict(
    service({ webhook_url: '', webhooks_from_rest_enabled: false }));
  assert.equal(state, 'no-url');
  assert.match(detail, /54051/);
});

test('plain http is rejected and insecure', () => {
  const [state, detail] = verdict(service({ webhook_url: 'http://app.example.com/sync' }));
  assert.equal(state, 'insecure');
  assert.match(detail, /in the clear/);
});

test('a url with no scheme is not absolute', () => {
  assert.equal(verdict(service({ webhook_url: 'app.example.com/sync' }))[0],
               'not-absolute');
});

test('alerts against a well formed url mean unreachable', () => {
  const [state, detail] = verdict(service(), false, 12);
  assert.equal(state, 'unreachable');
  assert.match(detail, /12 alert\(s\)/);
});

test('a healthy service is ok', () => {
  assert.equal(verdict(service(), true)[0], 'ok');
});

test('alertCounts coerce the code and key on the resource', () => {
  const alerts = [{ error_code: '54051', resource_sid: 'IS1' },
                  { error_code: 54051, resource_sid: 'IS1' },
                  { error_code: 54051, resource_sid: 'IS2' },
                  { error_code: 11200, resource_sid: 'IS1' },
                  { error_code: null, resource_sid: 'IS1' }];
  assert.deepEqual([...alertCounts(alerts).entries()], [['IS1', 2], ['IS2', 1]]);
});

test('an alert with no resource is still counted', () => {
  assert.deepEqual([...alertCounts([{ error_code: 54051 }]).entries()],
                   [['(unattributed)', 1]]);
});