incomplete_expired volume means confirmation is broken

"""Measure Stripe incomplete_expired subscriptions against activations.

Read only. Two paginated GETs, no writes: give this a RESTRICTED key with read
access to Subscriptions. The repair is printed, never performed, because this
script holds a credential to a live payments account.
"""
import argparse
import logging
import os
import sys
import time

import requests

logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s")
log = logging.getLogger("stripe_incomplete_expired_rate")

API = "https://api.stripe.com/v1"

# Share of activations above which the confirmation step is failing for part of
# the traffic rather than being abandoned by part of the customers.
LEAKING = 0.10
# Above this, it is not a slice of traffic any more.
BROKEN = 0.50


def verdict(expired, active, days=30, leaking=LEAKING, broken=BROKEN):
    """Judge one window of signups by the share that never confirmed. Pure.

    Takes two counts rather than one subscription, because the finding is a
    ratio: 200 expired subscriptions is background noise against 4,000
    activations and an outage against 300, and no single row can tell you which.

    Returns (state, detail).
    """
    if expired < 0 or active < 0:
        return ("unknown", "negative counts, so the ratio means nothing")

    if expired == 0 and active == 0:
        return ("no-signups",
                "no subscriptions created in the last %d day(s), so there is "
                "nothing to measure" % days)

    if expired == 0:
        return ("clean",
                "%d activation(s) in %d day(s) and nothing expired unconfirmed"
                % (active, days))

    if active == 0:
        return ("broken",
                "%d subscription(s) expired unconfirmed and not one activated in "
                "%d day(s): nothing is confirming at all" % (expired, days))

    ratio = expired / active
    pct = 100.0 * ratio

    if ratio >= broken:
        return ("broken",
                "%d expired against %d activation(s), %.1f%%: the confirmation "
                "step is failing for most of the traffic" % (expired, active, pct))

    if ratio >= leaking:
        return ("leaking",
                "%d expired against %d activation(s), %.1f%%: a slice of the "
                "traffic cannot complete the confirmation" % (expired, active, pct))

    return ("background",
            "%d expired against %d activation(s), %.1f%%: ordinary abandonment"
            % (expired, active, pct))


def get(session, path, **params):
    r = session.get(API + path, params=params, timeout=30)
    if r.status_code == 401:
        raise SystemExit("401 from Stripe: the key is wrong, or is for the other mode")
    r.raise_for_status()
    return r.json()


def count(session, status, since, limit):
    """Page a status to the end. One page of each and a division is a wrong answer."""
    total = 0
    params = {"status": status, "limit": 100, "created[gte]": since}
    while True:
        page = get(session, "/subscriptions", **params)
        rows = page.get("data", [])
        total += len(rows)
        if not page.get("has_more") or not rows or total >= limit:
            break
        params["starting_after"] = rows[-1]["id"]
    return total


def main():
    ap = argparse.ArgumentParser(description=__doc__)
    ap.add_argument("--days", type=int, default=30,
                    help="window to measure, in days (default 30)")
    ap.add_argument("--max-rows", type=int, default=5000,
                    help="stop paginating each status after this many rows")
    args = ap.parse_args()

    key = os.environ.get("STRIPE_API_KEY")
    if not key:
        log.error("set STRIPE_API_KEY (use a restricted, read-only key)")
        return 2

    s = requests.Session()
    s.headers.update({"Authorization": "Bearer " + key})

    since = int(time.time()) - args.days * 86400
    expired = count(s, "incomplete_expired", since, args.max_rows)
    active = count(s, "active", since, args.max_rows)

    state, detail = verdict(expired, active, args.days)
    if state in ("clean", "background", "no-signups"):
        log.info("%s: %s", state, detail)
        return 0

    log.warning("%s: %s", state, detail)
    log.warning("repair: create with payment_behavior=default_incomplete, expand "
                "latest_invoice.confirmation_secret, and confirm it client side "
                "in the same session")
    log.warning("repair: handle invoice.payment_action_required so an unfinished "
                "signup gets an email rather than a countdown")
    log.warning("note: expired subscriptions are terminal. The invoice is void "
                "and no API call revives them; these customers need a new signup.")
    return 1


if __name__ == "__main__":
    sys.exit(main())

/**
 * Measure Stripe incomplete_expired subscriptions against activations.
 *
 * Read only. Two paginated GETs, no writes: give this a RESTRICTED key with read
 * access to Subscriptions. The repair is printed, never performed.
 */
const API = 'https://api.stripe.com/v1';

// Share of activations above which the confirmation step is failing for part of
// the traffic rather than being abandoned by part of the customers.
export const LEAKING = 0.10;
export const BROKEN = 0.50;

/**
 * Judge one window of signups by the share that never confirmed. Pure.
 *
 * Takes two counts rather than one subscription, because the finding is a ratio:
 * 200 expired is noise against 4,000 activations and an outage against 300.
 */
export function verdict(expired, active, days = 30, leaking = LEAKING, broken = BROKEN) {
  if (expired < 0 || active < 0) {
    return ['unknown', 'negative counts, so the ratio means nothing'];
  }

  if (expired === 0 && active === 0) {
    return ['no-signups',
      `no subscriptions created in the last ${days} day(s), so there is nothing to measure`];
  }

  if (expired === 0) {
    return ['clean',
      `${active} activation(s) in ${days} day(s) and nothing expired unconfirmed`];
  }

  if (active === 0) {
    return ['broken',
      `${expired} subscription(s) expired unconfirmed and not one activated in ` +
      `${days} day(s): nothing is confirming at all`];
  }

  const ratio = expired / active;
  const pct = (100 * ratio).toFixed(1);

  if (ratio >= broken) {
    return ['broken',
      `${expired} expired against ${active} activation(s), ${pct}%: the ` +
      'confirmation step is failing for most of the traffic'];
  }

  if (ratio >= leaking) {
    return ['leaking',
      `${expired} expired against ${active} activation(s), ${pct}%: a slice of ` +
      'the traffic cannot complete the confirmation'];
  }

  return ['background',
    `${expired} expired against ${active} activation(s), ${pct}%: ordinary abandonment`];
}

async function get(key, path, params = {}) {
  const url = new URL(API + path);
  for (const [k, v] of Object.entries(params)) url.searchParams.set(k, v);
  const res = await fetch(url, { headers: { Authorization: `Bearer ${key}` } });
  if (res.status === 401) {
    throw new Error('401 from Stripe: the key is wrong, or is for the other mode');
  }
  if (!res.ok) throw new Error(`${res.status} from ${url.pathname}`);
  return res.json();
}

// Page a status to the end. One page of each and a division is a wrong answer.
async function count(key, status, since, limit) {
  let total = 0;
  const params = { status, limit: 100, 'created[gte]': since };
  for (;;) {
    const page = await get(key, '/subscriptions', params);
    const rows = page.data ?? [];
    total += rows.length;
    if (!page.has_more || rows.length === 0 || total >= limit) break;
    params.starting_after = rows[rows.length - 1].id;
  }
  return total;
}

async function main() {
  const days = Number(process.argv[2] ?? 30);

  const key = process.env.STRIPE_API_KEY;
  if (!key) {
    console.error('set STRIPE_API_KEY (use a restricted, read-only key)');
    process.exitCode = 2;
    return;
  }

  const since = Math.floor(Date.now() / 1000) - days * 86400;
  const expired = await count(key, 'incomplete_expired', since, 5000);
  const active = await count(key, 'active', since, 5000);

  const [state, detail] = verdict(expired, active, days);
  if (state === 'clean' || state === 'background' || state === 'no-signups') {
    console.log(`${state}: ${detail}`);
    return;
  }

  console.warn(`${state}: ${detail}`);
  console.warn('repair: create with payment_behavior=default_incomplete, expand ' +
    'latest_invoice.confirmation_secret, and confirm it client side in the same session');
  console.warn('repair: handle invoice.payment_action_required so an unfinished ' +
    'signup gets an email rather than a countdown');
  console.warn('note: expired subscriptions are terminal. The invoice is void and ' +
    'no API call revives them; these customers need a new signup.');
  process.exitCode = 1;
}

if (import.meta.url === `file://${process.argv[1]}`) {
  main().catch((err) => { console.error(err.message); process.exitCode = 2; });
}

from stripe_incomplete_expired_rate import verdict


def test_the_same_count_is_noise_against_enough_activations():
    state, detail = verdict(20, 4000)
    assert state == "background"
    assert "abandonment" in detail


def test_the_same_count_is_a_leak_against_a_small_one():
    # 20 of 150 is 13% of activations: a slice of traffic cannot confirm.
    state, detail = verdict(20, 150)
    assert state == "leaking"
    assert "13.3%" in detail


def test_expired_with_no_activations_does_not_divide_by_zero():
    state, detail = verdict(31, 0, days=14)
    assert state == "broken"
    assert "not one activated" in detail


def test_a_quiet_window_is_not_reported_as_healthy_signups():
    assert verdict(0, 0)[0] == "no-signups"


def test_nothing_expired_is_clean():
    assert verdict(0, 900)[0] == "clean"

import { test } from 'node:test';
import assert from 'node:assert/strict';
import { verdict } from './stripe-incomplete-expired-rate.mjs';

test('the same count is noise against enough activations', () => {
  const [state, detail] = verdict(20, 4000);
  assert.equal(state, 'background');
  assert.match(detail, /abandonment/);
});

test('the same count is a leak against a small one', () => {
  const [state, detail] = verdict(20, 150);
  assert.equal(state, 'leaking');
  assert.match(detail, /13\.3%/);
});

test('expired with no activations does not divide by zero', () => {
  const [state, detail] = verdict(31, 0, 14);
  assert.equal(state, 'broken');
  assert.match(detail, /not one activated/);
});

test('a quiet window is not reported as healthy signups', () => {
  assert.equal(verdict(0, 0)[0], 'no-signups');
});

test('nothing expired is clean', () => {
  assert.equal(verdict(0, 900)[0], 'clean');
});