recordings billed for storage until something deletes them

"""Report Twilio call recordings that are still billing for storage.

The finding is the money, not the file count: a count of recordings is a number
nobody can price, and the accumulated spend plus a year's projection is the same
fact in the units that get a retention job written.

Read only. GET requests and nothing else: give this an API Key with read access
rather than the account auth token. The repair is printed, never performed,
because this script holds a credential to an account that can send messages and
spend money.
"""
import argparse
import datetime
import email.utils
import logging
import os
import sys

import requests

logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s")
log = logging.getLogger("twilio_recording_storage_audit")

HOST = "https://api.twilio.com"
BASE = HOST + "/2010-04-01"

CATEGORY = "recordings"


def parse_created(value):
    """A Twilio date_created as a date, or None.

    The 2010-04-01 API returns RFC 2822 dates (Tue, 18 Apr 2023 09:12:00 +0000),
    not ISO 8601. Handing one of those to a date parser expecting ISO gives you
    an exception on every row, which reads exactly like an empty account.
    """
    try:
        return email.utils.parsedate_to_datetime(str(value)).date()
    except (TypeError, ValueError, AttributeError):
        return None


def older_than(recordings, window_days, today):
    """(how many are past the window, age in days of the oldest). Pure."""
    ages = []
    for recording in recordings or []:
        created = parse_created(recording.get("date_created"))
        if created is None:
            continue
        ages.append((today - created).days)
    if not ages:
        return (0, None)
    return (len([a for a in ages if a > window_days]), max(ages))


def stored_minutes(recordings):
    """Minutes of media in the sample. Duration is seconds, as a string. Pure."""
    total = 0.0
    for recording in recordings or []:
        try:
            total += float(recording.get("duration"))
        except (TypeError, ValueError):
            continue
    return round(total / 60.0, 1)


def daily_rate(records):
    """Mean priced day out of a Usage/Records/Daily page. Pure.

    The mean rather than the median here: storage accrues every day at a rate
    set by the size of the pile, so there is no spiky day to defend against and
    the mean is the honest per-day figure to project from.
    """
    prices = []
    for record in records or []:
        try:
            prices.append(max(0.0, float(record.get("price"))))
        except (TypeError, ValueError):
            continue
    if not prices:
        return 0.0
    return sum(prices) / len(prices)


def project(rate, days=365):
    """What the current rate costs over a horizon. Pure."""
    return round((rate or 0.0) * days, 2)


def verdict(total_price, rate, stale_count, sample_size, window_days):
    """Classify the storage position. Pure, so the arithmetic that turns a pile
    of files into a number somebody will act on is testable offline.

    Returns (state, detail).
    """
    total_price = total_price or 0.0

    if sample_size <= 0:
        if total_price <= 0:
            return ("empty",
                    "no recordings and nothing billed to recording storage: there "
                    "is nothing here to release.")
        return ("billed-only",
                "no recordings stored now, but %.2f billed to recording storage "
                "historically: the spend is in the past and the pile is gone."
                % total_price)

    if stale_count == 0:
        return ("retained",
                "%d recording(s) sampled, none older than %d days, %.2f billed to "
                "recording storage so far: something is deleting them."
                % (sample_size, window_days, total_price))

    if rate > 0:
        return ("accumulating",
                "%d of %d sampled recording(s) older than %d days. %.2f billed to "
                "recording storage to date, running at %.2f a day: about %.2f more "
                "over the next year unless something deletes them."
                % (stale_count, sample_size, window_days, total_price, rate,
                   project(rate)))

    return ("unpriced",
            "%d of %d sampled recording(s) older than %d days, and no priced usage "
            "in the window: the media is still stored, so check the category name "
            "on your usage report and re-run with --category."
            % (stale_count, sample_size, window_days))


def get(session, url, **params):
    r = session.get(url, params=params, timeout=30)
    if r.status_code in (401, 403):
        raise SystemExit("%d from Twilio: check TWILIO_ACCOUNT_SID and that the "
                         "API key belongs to that account with read access"
                         % r.status_code)
    r.raise_for_status()
    return r.json()


def list_recordings(session, account, limit=2000):
    """Page Recordings.json. next_page_uri is a path, not an absolute URL."""
    url = "%s/Accounts/%s/Recordings.json" % (BASE, account)
    params = {"PageSize": 1000}
    out = []
    while url and len(out) < limit:
        page = get(session, url, **params)
        out.extend(page.get("recordings", []))
        nxt = page.get("next_page_uri")
        url, params = (HOST + nxt) if nxt else None, {}
    return out[:limit]


def all_time_spend(session, account, category):
    """Accumulated (price, usage, price_unit) for one usage category."""
    page = get(session, "%s/Accounts/%s/Usage/Records/AllTime.json" % (BASE, account),
               Category=category, PageSize=1)
    rows = page.get("usage_records", [])
    if not rows:
        return (0.0, "0", "")
    row = rows[0]
    try:
        price = float(row.get("price"))
    except (TypeError, ValueError):
        price = 0.0
    return (price, row.get("usage", "0"), row.get("price_unit", ""))


def daily_records(session, account, category, days):
    start = (datetime.date.today() - datetime.timedelta(days=days)).isoformat()
    page = get(session, "%s/Accounts/%s/Usage/Records/Daily.json" % (BASE, account),
               Category=category, StartDate=start, PageSize=100)
    return page.get("usage_records", [])


def main():
    ap = argparse.ArgumentParser(description=__doc__)
    ap.add_argument("--window-days", type=int, default=90,
                    help="retention window: anything older is evidence that "
                         "nothing is deleting recordings")
    ap.add_argument("--days", type=int, default=30,
                    help="days of usage records the daily rate is taken from")
    ap.add_argument("--sample", type=int, default=2000,
                    help="how many recordings to page through")
    ap.add_argument("--category", default=CATEGORY,
                    help="usage category carrying recording storage on your bill")
    args = ap.parse_args()

    account = os.environ.get("TWILIO_ACCOUNT_SID")
    key = os.environ.get("TWILIO_API_KEY")
    secret = os.environ.get("TWILIO_API_SECRET")
    if not (account and key and secret):
        log.error("set TWILIO_ACCOUNT_SID, TWILIO_API_KEY and TWILIO_API_SECRET "
                  "(an API Key with read access, not the auth token)")
        return 2

    session = requests.Session()
    session.auth = (key, secret)

    recordings = list_recordings(session, account, args.sample)
    total_price, usage, unit = all_time_spend(session, account, args.category)
    rate = daily_rate(daily_records(session, account, args.category, args.days))
    stale, oldest = older_than(recordings, args.window_days, datetime.date.today())

    log.info("%d recording(s) sampled, %s stored minute(s), %.2f %s billed to %s "
             "all time", len(recordings), stored_minutes(recordings), total_price,
             unit, args.category)
    if oldest is not None:
        log.info("oldest recording in the sample: %d days old", oldest)

    state, detail = verdict(total_price, rate, stale, len(recordings),
                            args.window_days)
    if state in ("empty", "retained", "billed-only"):
        log.info("%-14s %s", state, detail)
        return 0

    log.warning("%-14s %s", state, detail)
    log.warning("  repair: for each recording already archived on your side, "
                "delete it from %s/Accounts/%s/Recordings/{RecordingSid}.json "
                "after verifying the copy you hold", BASE, account)
    log.warning("  then set a retention policy in Console > Voice > Settings so "
                "the next four years do not repeat this one")
    log.warning("  the API has no field saying which recordings you have "
                "archived: that match is yours to make, which is why this "
                "script only reports")
    return 1


if __name__ == "__main__":
    sys.exit(main())

/**
 * Report Twilio call recordings that are still billing for storage.
 *
 * The finding is the money, not the file count. Read only: GET requests and
 * nothing else, with the repair printed rather than performed.
 */
const HOST = 'https://api.twilio.com';
const BASE = `${HOST}/2010-04-01`;

const CATEGORY = 'recordings';

/**
 * A Twilio date_created as a date at UTC midnight, or null. The 2010-04-01 API
 * returns RFC 2822 dates rather than ISO 8601, and a parser that assumes ISO
 * fails on every row, which reads exactly like an empty account.
 */
export function parseCreated(value) {
  if (value === null || value === undefined) return null;
  const parsed = new Date(String(value));
  if (Number.isNaN(parsed.getTime())) return null;
  return new Date(Date.UTC(parsed.getUTCFullYear(), parsed.getUTCMonth(),
                           parsed.getUTCDate()));
}

/** [how many are past the window, age in days of the oldest]. Pure. */
export function olderThan(recordings, windowDays, today) {
  const ages = [];
  for (const recording of recordings ?? []) {
    const created = parseCreated(recording?.date_created);
    if (created === null) continue;
    ages.push(Math.floor((today.getTime() - created.getTime()) / 86400000));
  }
  if (!ages.length) return [0, null];
  return [ages.filter((a) => a > windowDays).length, Math.max(...ages)];
}

/** Minutes of media in the sample. duration is seconds, as a string. Pure. */
export function storedMinutes(recordings) {
  let total = 0;
  for (const recording of recordings ?? []) {
    const seconds = Number.parseFloat(recording?.duration ?? '');
    if (Number.isFinite(seconds)) total += seconds;
  }
  return Math.round((total / 60) * 10) / 10;
}

/** Mean priced day out of a Usage/Records/Daily page. Pure. */
export function dailyRate(records) {
  const prices = [];
  for (const record of records ?? []) {
    const price = Number.parseFloat(record?.price ?? '');
    if (Number.isFinite(price)) prices.push(Math.max(0, price));
  }
  if (!prices.length) return 0;
  return prices.reduce((a, b) => a + b, 0) / prices.length;
}

/** What the current rate costs over a horizon. Pure. */
export function project(rate, days = 365) {
  return Math.round((rate || 0) * days * 100) / 100;
}

/**
 * Classify the storage position. Pure, so the arithmetic that turns a pile of
 * files into a number somebody will act on is testable offline.
 * Returns [state, detail].
 */
export function verdict(totalPrice, rate, staleCount, sampleSize, windowDays) {
  const total = totalPrice || 0;

  if (sampleSize <= 0) {
    if (total <= 0) {
      return ['empty',
        'no recordings and nothing billed to recording storage: there is nothing ' +
        'here to release.'];
    }
    return ['billed-only',
      `no recordings stored now, but ${total.toFixed(2)} billed to recording ` +
      'storage historically: the spend is in the past and the pile is gone.'];
  }

  if (staleCount === 0) {
    return ['retained',
      `${sampleSize} recording(s) sampled, none older than ${windowDays} days, ` +
      `${total.toFixed(2)} billed to recording storage so far: something is ` +
      'deleting them.'];
  }

  if (rate > 0) {
    return ['accumulating',
      `${staleCount} of ${sampleSize} sampled recording(s) older than ${windowDays} ` +
      `days. ${total.toFixed(2)} billed to recording storage to date, running at ` +
      `${rate.toFixed(2)} a day: about ${project(rate).toFixed(2)} more over the ` +
      'next year unless something deletes them.'];
  }

  return ['unpriced',
    `${staleCount} of ${sampleSize} sampled recording(s) older than ${windowDays} ` +
    'days, and no priced usage in the window: the media is still stored, so check ' +
    'the category name on your usage report and re-run with --category.'];
}

function authHeader(key, secret) {
  return `Basic ${Buffer.from(`${key}:${secret}`).toString('base64')}`;
}

async function get(auth, url, params = {}) {
  const u = new URL(url);
  for (const [k, v] of Object.entries(params)) u.searchParams.set(k, v);
  const res = await fetch(u, { headers: { Authorization: auth } });
  if (res.status === 401 || res.status === 403) {
    throw new Error(`${res.status} from Twilio: check TWILIO_ACCOUNT_SID and ` +
                    'that the API key belongs to that account with read access');
  }
  if (!res.ok) throw new Error(`${res.status} from ${u.pathname}`);
  return res.json();
}

export async function listRecordings(auth, account, limit = 2000) {
  let url = `${BASE}/Accounts/${account}/Recordings.json`;
  let params = { PageSize: 1000 };
  const out = [];
  while (url && out.length < limit) {
    const page = await get(auth, url, params);
    out.push(...(page.recordings ?? []));
    url = page.next_page_uri ? HOST + page.next_page_uri : null;
    params = {};
  }
  return out.slice(0, limit);
}

async function allTimeSpend(auth, account, category) {
  const page = await get(auth, `${BASE}/Accounts/${account}/Usage/Records/AllTime.json`,
                         { Category: category, PageSize: 1 });
  const rows = page.usage_records ?? [];
  if (!rows.length) return [0, '0', ''];
  const price = Number.parseFloat(rows[0].price ?? '');
  return [Number.isFinite(price) ? price : 0, rows[0].usage ?? '0',
          rows[0].price_unit ?? ''];
}

async function dailyRecords(auth, account, category, days) {
  const start = new Date(Date.now() - days * 86400000).toISOString().slice(0, 10);
  const page = await get(auth, `${BASE}/Accounts/${account}/Usage/Records/Daily.json`,
                         { Category: category, StartDate: start, PageSize: 100 });
  return page.usage_records ?? [];
}

async function main() {
  const account = process.env.TWILIO_ACCOUNT_SID;
  const key = process.env.TWILIO_API_KEY;
  const secret = process.env.TWILIO_API_SECRET;
  if (!account || !key || !secret) {
    console.error('set TWILIO_ACCOUNT_SID, TWILIO_API_KEY and TWILIO_API_SECRET ' +
                  '(an API Key with read access, not the auth token)');
    process.exitCode = 2;
    return;
  }
  const auth = authHeader(key, secret);

  const arg = (name, fallback) => {
    const i = process.argv.indexOf(name);
    return i === -1 ? fallback : Number.parseFloat(process.argv[i + 1]);
  };
  const windowDays = arg('--window-days', 90);
  const days = arg('--days', 30);
  const sample = arg('--sample', 2000);
  const ci = process.argv.indexOf('--category');
  const category = ci === -1 ? CATEGORY : process.argv[ci + 1];

  const recordings = await listRecordings(auth, account, sample);
  const [totalPrice, , unit] = await allTimeSpend(auth, account, category);
  const rate = dailyRate(await dailyRecords(auth, account, category, days));
  const now = new Date();
  const today = new Date(Date.UTC(now.getUTCFullYear(), now.getUTCMonth(),
                                  now.getUTCDate()));
  const [stale, oldest] = olderThan(recordings, windowDays, today);

  console.log(`${recordings.length} recording(s) sampled, ` +
              `${storedMinutes(recordings)} stored minute(s), ` +
              `${totalPrice.toFixed(2)} ${unit} billed to ${category} all time`);
  if (oldest !== null) {
    console.log(`oldest recording in the sample: ${oldest} days old`);
  }

  const [state, detail] = verdict(totalPrice, rate, stale, recordings.length,
                                  windowDays);
  if (['empty', 'retained', 'billed-only'].includes(state)) {
    console.log(`${state.padEnd(14)} ${detail}`);
    return;
  }

  console.warn(`${state.padEnd(14)} ${detail}`);
  console.warn('  repair: for each recording already archived on your side, delete ' +
               `it from ${BASE}/Accounts/${account}/Recordings/{RecordingSid}.json ` +
               'after verifying the copy you hold');
  console.warn('  then set a retention policy in Console > Voice > Settings so the ' +
               'next four years do not repeat this one');
  console.warn('  the API has no field saying which recordings you have archived: ' +
               'that match is yours to make, which is why this script only reports');
  process.exitCode = 1;
}

// Only run when invoked directly. The test file imports this module, and without
// the guard main() would run there too, fail on the missing credentials and set a
// non-zero exit code that fails the whole test file even as every test passes.
if (import.meta.url === `file://${process.argv[1]}`) {
  main().catch((err) => { console.error(err.message); process.exitCode = 2; });
}

import datetime

from twilio_recording_storage_audit import (daily_rate, older_than, parse_created,
                                            project, stored_minutes, verdict)

TODAY = datetime.date(2026, 8, 30)


def rec(created, duration="120"):
    return {"sid": "RE01", "date_created": created, "duration": duration}


def test_rfc_2822_dates_parse_and_iso_style_junk_does_not():
    assert parse_created("Tue, 18 Apr 2023 09:12:00 +0000") == datetime.date(2023, 4, 18)
    assert parse_created("not a date") is None
    assert parse_created(None) is None


def test_ages_are_measured_against_the_day_you_pass_in():
    stale, oldest = older_than([rec("Mon, 01 Jun 2026 00:00:00 +0000"),
                                rec("Sat, 01 Jun 2024 00:00:00 +0000")], 90, TODAY)
    assert stale == 1
    assert oldest == 820


def test_an_unparseable_row_is_skipped_rather_than_counted_as_new():
    stale, oldest = older_than([rec("garbage")], 90, TODAY)
    assert (stale, oldest) == (0, None)


def test_stored_minutes_add_up_and_ignore_bad_durations():
    assert stored_minutes([rec("x", "90"), rec("x", "30"), rec("x", None)]) == 2.0


def test_the_daily_rate_is_the_mean_of_the_priced_days():
    assert daily_rate([{"price": "1.00"}, {"price": "3.00"}]) == 2.0
    assert daily_rate([]) == 0.0


def test_the_projection_is_the_rate_over_a_year():
    assert project(0.5) == 182.5
    assert project(0.0) == 0.0


def test_no_recordings_and_no_spend_is_nothing_to_do():
    state, _ = verdict(0.0, 0.0, 0, 0, 90)
    assert state == "empty"


def test_historic_spend_with_nothing_stored_is_not_a_finding():
    state, detail = verdict(400.0, 0.0, 0, 0, 90)
    assert state == "billed-only"
    assert "in the past" in detail


def test_a_working_retention_job_reads_as_retained():
    state, detail = verdict(812.44, 0.4, 0, 1204, 90)
    assert state == "retained"
    assert "something is deleting them" in detail


def test_the_finding_is_the_projected_cost_not_the_file_count():
    state, detail = verdict(3200.0, 2.0, 38000, 40000, 90)
    assert state == "accumulating"
    assert "730.00 more over the next year" in detail


def test_stale_files_with_no_priced_usage_send_you_to_the_category_name():
    state, detail = verdict(0.0, 0.0, 12, 40, 90)
    assert state == "unpriced"
    assert "--category" in detail

import { test } from 'node:test';
import assert from 'node:assert/strict';
import { dailyRate, olderThan, parseCreated, project, storedMinutes, verdict }
  from './twilio-recording-storage-audit.mjs';

const TODAY = new Date(Date.UTC(2026, 7, 30));

const rec = (created, duration = '120') => ({
  sid: 'RE01', date_created: created, duration,
});

test('RFC 2822 dates parse and junk does not', () => {
  assert.equal(parseCreated('Tue, 18 Apr 2023 09:12:00 +0000').toISOString(),
               '2023-04-18T00:00:00.000Z');
  assert.equal(parseCreated('not a date'), null);
  assert.equal(parseCreated(null), null);
});

test('ages are measured against the day you pass in', () => {
  const [stale, oldest] = olderThan([rec('Mon, 01 Jun 2026 00:00:00 +0000'),
                                     rec('Sat, 01 Jun 2024 00:00:00 +0000')],
                                    90, TODAY);
  assert.equal(stale, 1);
  assert.equal(oldest, 820);
});

test('an unparseable row is skipped rather than counted as new', () => {
  assert.deepEqual(olderThan([rec('garbage')], 90, TODAY), [0, null]);
});

test('stored minutes add up and ignore bad durations', () => {
  assert.equal(storedMinutes([rec('x', '90'), rec('x', '30'), rec('x', null)]), 2.0);
});

test('the daily rate is the mean of the priced days', () => {
  assert.equal(dailyRate([{ price: '1.00' }, { price: '3.00' }]), 2.0);
  assert.equal(dailyRate([]), 0);
});

test('the projection is the rate over a year', () => {
  assert.equal(project(0.5), 182.5);
  assert.equal(project(0), 0);
});

test('no recordings and no spend is nothing to do', () => {
  assert.equal(verdict(0, 0, 0, 0, 90)[0], 'empty');
});

test('historic spend with nothing stored is not a finding', () => {
  const [state, detail] = verdict(400, 0, 0, 0, 90);
  assert.equal(state, 'billed-only');
  assert.match(detail, /in the past/);
});

test('a working retention job reads as retained', () => {
  const [state, detail] = verdict(812.44, 0.4, 0, 1204, 90);
  assert.equal(state, 'retained');
  assert.match(detail, /something is deleting them/);
});

test('the finding is the projected cost, not the file count', () => {
  const [state, detail] = verdict(3200, 2, 38000, 40000, 90);
  assert.equal(state, 'accumulating');
  assert.match(detail, /730\.00 more over the next year/);
});

test('stale files with no priced usage send you to the category name', () => {
  const [state, detail] = verdict(0, 0, 12, 40, 90);
  assert.equal(state, 'unpriced');
  assert.match(detail, /--category/);
});