Slack disabled event delivery and will not turn it back on

"""Find channels where a Slack app is addressed and has stopped answering.

Read only. GET requests and nothing else: channels:history and membership are
enough. This detects the symptom of disabled event delivery, not the flag: no
read method reports whether Slack is delivering, so the repair ends at the app
configuration page and is printed, never performed.
"""
import argparse
import logging
import os
import sys

import requests

logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s")
log = logging.getLogger("slack_event_silence_audit")

API = "https://slack.com/api/"


def scan(messages, bot_id, bot_user_id):
    """Reduce one page of history to the four numbers that matter. Pure.

    A trigger is a message mentioning the bot that the bot did not write. A
    reply is any message carrying the app's own bot_id. `unanswered` counts the
    triggers that arrived after the app last said anything.
    """
    mention = "<@%s>" % bot_user_id
    replies, triggers = [], []
    for m in messages:
        ts = float(m.get("ts") or 0)
        if m.get("bot_id") == bot_id:
            replies.append(ts)
        elif mention in (m.get("text") or ""):
            triggers.append(ts)
    last_reply = max(replies) if replies else None
    last_trigger = max(triggers) if triggers else None
    unanswered = len([t for t in triggers if last_reply is None or t > last_reply])
    return {"replies": len(replies), "triggers": len(triggers),
            "last_reply": last_reply, "last_trigger": last_trigger,
            "unanswered": unanswered}


def verdict(stats, min_triggers=3):
    """Decide whether the silence is evidence. Pure, and mostly a refusal.

    Three different causes produce this shape - delivery disabled by Slack, the
    handler down, and events never subscribed to - and none of them can be told
    apart from inside the workspace. The states name the shape, not the cause.
    """
    if not stats["triggers"]:
        return ("no-triggers",
                "nothing addressed the app in this window, so there is no "
                "evidence either way. Silence is not a finding on its own.")
    if not stats["unanswered"]:
        return ("answering",
                "%d mention(s), and the app replied after the most recent one"
                % stats["triggers"])
    if not stats["replies"]:
        return ("never-answered",
                "%d mention(s) and the app has never posted here. That points at "
                "subscriptions never configured or a Request URL that never "
                "verified, rather than at delivery being switched off."
                % stats["triggers"])
    if stats["unanswered"] >= min_triggers:
        hours = (stats["last_trigger"] - stats["last_reply"]) / 3600.0
        return ("silent",
                "%d mention(s) since the app last replied, spanning %.1f hour(s). "
                "It was answering and then stopped: check whether Slack disabled "
                "event delivery." % (stats["unanswered"], hours))
    return ("too-little-evidence",
            "%d unanswered mention(s), below the %d needed to call it. People "
            "type a bot's name without expecting an answer."
            % (stats["unanswered"], min_triggers))


def get(session, method, **params):
    r = session.get(API + method, params=params, timeout=30)
    try:
        return r.json()
    except ValueError:
        return {"ok": False, "error": "unparseable_body"}


def main():
    ap = argparse.ArgumentParser(description=__doc__)
    ap.add_argument("channels", nargs="+", help="channel IDs the app serves (C...)")
    ap.add_argument("--limit", type=int, default=200,
                    help="messages of history per channel (default 200)")
    ap.add_argument("--min-triggers", type=int, default=3,
                    help="unanswered mentions before it counts (default 3)")
    args = ap.parse_args()

    token = os.environ.get("SLACK_BOT_TOKEN")
    if not token:
        log.error("set SLACK_BOT_TOKEN (channels:history and membership are enough)")
        return 2

    s = requests.Session()
    s.headers.update({"Authorization": "Bearer " + token})

    me = get(s, "auth.test")
    if me.get("ok") is not True:
        log.error("auth.test answered 200 with ok: false, error=%s", me.get("error"))
        return 2
    bot_id, bot_user = me.get("bot_id"), me.get("user_id")
    log.info("app is %s (bot_id=%s, mentioned as <@%s>)", me.get("user"), bot_id, bot_user)

    bad = 0
    for cid in args.channels:
        body = get(s, "conversations.history", channel=cid, limit=str(args.limit))
        if body.get("ok") is not True:
            bad += 1
            log.warning("%-20s %-12s history refused: error=%s. Membership and "
                        "channels:history come first; this audit assumes both",
                        "unreadable", cid, body.get("error"))
            continue
        stats = scan(body.get("messages") or [], bot_id, bot_user)
        state, detail = verdict(stats, args.min_triggers)
        line = "%-20s %-12s %s" % (state, cid, detail)
        if state in ("silent", "never-answered"):
            bad += 1
            log.warning(line)
            log.warning("  the Web API cannot tell you whether Slack disabled "
                        "delivery: open Event Subscriptions in the app config")
            log.warning("  repair: fix the endpoint, re-enable delivery by hand, then "
                        "alert on the Request URL before 95%% of an hour fails")
        else:
            log.info(line)

    log.info("%d channel(s) checked, %d where the app has gone quiet",
             len(args.channels), bad)
    return 1 if bad else 0


if __name__ == "__main__":
    sys.exit(main())

/**
 * Find channels where a Slack app is addressed and has stopped answering.
 *
 * Read only. GET requests and nothing else: channels:history and membership are
 * enough. This detects the symptom of disabled event delivery, not the flag: no
 * read method reports whether Slack is delivering, so the repair ends at the app
 * configuration page and is printed, never performed.
 */
const API = 'https://slack.com/api/';

/**
 * Reduce one page of history to the four numbers that matter. Pure.
 * A trigger is a message mentioning the bot that the bot did not write; a reply
 * is any message carrying the app's own bot_id.
 */
export function scan(messages, botId, botUserId) {
  const mention = `<@${botUserId}>`;
  const replies = [];
  const triggers = [];
  for (const m of messages) {
    const ts = Number(m.ts ?? 0);
    if (m.bot_id === botId) replies.push(ts);
    else if ((m.text ?? '').includes(mention)) triggers.push(ts);
  }
  const lastReply = replies.length ? Math.max(...replies) : null;
  const lastTrigger = triggers.length ? Math.max(...triggers) : null;
  const unanswered = triggers.filter((t) => lastReply === null || t > lastReply).length;
  return {
    replies: replies.length, triggers: triggers.length,
    lastReply, lastTrigger, unanswered,
  };
}

/**
 * Decide whether the silence is evidence. Pure, and mostly a refusal.
 * Delivery disabled, the handler down and events never subscribed to all produce
 * this shape, so the states name the shape and not the cause.
 */
export function verdict(stats, minTriggers = 3) {
  if (!stats.triggers) {
    return ['no-triggers',
      'nothing addressed the app in this window, so there is no evidence either ' +
      'way. Silence is not a finding on its own.'];
  }
  if (!stats.unanswered) {
    return ['answering',
      `${stats.triggers} mention(s), and the app replied after the most recent one`];
  }
  if (!stats.replies) {
    return ['never-answered',
      `${stats.triggers} mention(s) and the app has never posted here. That points ` +
      'at subscriptions never configured or a Request URL that never verified, ' +
      'rather than at delivery being switched off.'];
  }
  if (stats.unanswered >= minTriggers) {
    const hours = (stats.lastTrigger - stats.lastReply) / 3600;
    return ['silent',
      `${stats.unanswered} mention(s) since the app last replied, spanning ` +
      `${hours.toFixed(1)} hour(s). It was answering and then stopped: check ` +
      'whether Slack disabled event delivery.'];
  }
  return ['too-little-evidence',
    `${stats.unanswered} unanswered mention(s), below the ${minTriggers} needed to ` +
    "call it. People type a bot's name without expecting an answer."];
}

async function get(token, method, params = {}) {
  const url = new URL(API + method);
  for (const [k, v] of Object.entries(params)) url.searchParams.set(k, v);
  const res = await fetch(url, { headers: { Authorization: `Bearer ${token}` } });
  try {
    return await res.json();
  } catch {
    return { ok: false, error: 'unparseable_body' };
  }
}

async function main() {
  const token = process.env.SLACK_BOT_TOKEN;
  if (!token) {
    console.error('set SLACK_BOT_TOKEN (channels:history and membership are enough)');
    process.exitCode = 2;
    return;
  }

  const argv = process.argv.slice(2);
  const li = argv.indexOf('--limit');
  const mi = argv.indexOf('--min-triggers');
  const limit = li === -1 ? '200' : argv[li + 1];
  const minTriggers = mi === -1 ? 3 : Number(argv[mi + 1]);
  const channels = argv.filter((a, n) => !a.startsWith('--')
    && argv[n - 1] !== '--limit' && argv[n - 1] !== '--min-triggers');

  if (!channels.length) {
    console.error('usage: node slack-event-silence-audit.mjs C0123ABCDEF [C...]');
    process.exitCode = 2;
    return;
  }

  const me = await get(token, 'auth.test');
  if (me.ok !== true) {
    console.error(`auth.test answered 200 with ok: false, error=${me.error}`);
    process.exitCode = 2;
    return;
  }
  const botId = me.bot_id;
  const botUser = me.user_id;
  console.log(`app is ${me.user} (bot_id=${botId}, mentioned as <@${botUser}>)`);

  let bad = 0;
  for (const cid of channels) {
    const body = await get(token, 'conversations.history', { channel: cid, limit });
    if (body.ok !== true) {
      bad += 1;
      console.warn(`${'unreadable'.padEnd(20)} ${cid.padEnd(12)} history refused: ` +
                   `error=${body.error}. Membership and channels:history come ` +
                   'first; this audit assumes both');
      continue;
    }
    const stats = scan(body.messages ?? [], botId, botUser);
    const [state, detail] = verdict(stats, minTriggers);
    const line = `${state.padEnd(20)} ${cid.padEnd(12)} ${detail}`;
    if (state === 'silent' || state === 'never-answered') {
      bad += 1;
      console.warn(line);
      console.warn('  the Web API cannot tell you whether Slack disabled delivery: ' +
                   'open Event Subscriptions in the app config');
      console.warn('  repair: fix the endpoint, re-enable delivery by hand, then ' +
                   'alert on the Request URL before 95% of an hour fails');
    } else {
      console.log(line);
    }
  }

  console.log(`${channels.length} channel(s) checked, ${bad} where the app has gone quiet`);
  process.exitCode = bad ? 1 : 0;
}

// Only run when invoked directly, so importing this module in the tests does not
// execute main() and fail the file on a missing token.
if (import.meta.url === `file://${process.argv[1]}`) {
  main().catch((err) => { console.error(err.message); process.exitCode = 2; });
}

from slack_event_silence_audit import scan, verdict

BOT = "B123"
BOT_USER = "U999"


def msg(ts, text="hello", bot=False):
    m = {"ts": "%d.000100" % ts, "text": text}
    if bot:
        m["bot_id"] = BOT
    return m


def mention(ts):
    return msg(ts, "<@%s> please deploy" % BOT_USER)


def test_scan_separates_triggers_from_replies():
    messages = [mention(100), msg(110, "unrelated chatter"), msg(120, "done", bot=True)]
    stats = scan(messages, BOT, BOT_USER)
    assert stats["triggers"] == 1
    assert stats["replies"] == 1
    assert stats["unanswered"] == 0


def test_the_bots_own_mention_of_itself_is_not_a_trigger():
    messages = [msg(100, "<@%s> was asked" % BOT_USER, bot=True)]
    assert scan(messages, BOT, BOT_USER)["triggers"] == 0


def test_a_run_of_mentions_after_the_last_reply_is_the_finding():
    messages = [msg(1000, "on it", bot=True), mention(5000), mention(9000), mention(13000)]
    state, detail = verdict(scan(messages, BOT, BOT_USER))
    assert state == "silent"
    assert "3 mention(s)" in detail


def test_an_app_that_never_replied_is_a_different_diagnosis():
    messages = [mention(1000), mention(2000), mention(3000), mention(4000)]
    state, detail = verdict(scan(messages, BOT, BOT_USER))
    assert state == "never-answered"
    assert "never configured" in detail


def test_a_reply_after_the_last_mention_is_healthy():
    messages = [mention(1000), msg(1100, "done", bot=True)]
    assert verdict(scan(messages, BOT, BOT_USER))[0] == "answering"


def test_one_unanswered_mention_is_not_enough():
    messages = [msg(1000, "done", bot=True), mention(2000)]
    assert verdict(scan(messages, BOT, BOT_USER))[0] == "too-little-evidence"


def test_a_quiet_channel_is_not_evidence():
    messages = [msg(1000, "morning"), msg(2000, "morning")]
    assert verdict(scan(messages, BOT, BOT_USER))[0] == "no-triggers"


def test_the_threshold_is_adjustable():
    messages = [msg(1000, "done", bot=True), mention(2000), mention(3000)]
    assert verdict(scan(messages, BOT, BOT_USER), min_triggers=2)[0] == "silent"

import { test } from 'node:test';
import assert from 'node:assert/strict';
import { scan, verdict } from './slack-event-silence-audit.mjs';

const BOT = 'B123';
const BOT_USER = 'U999';

function msg(ts, text = 'hello', bot = false) {
  const m = { ts: `${ts}.000100`, text };
  if (bot) m.bot_id = BOT;
  return m;
}

const mention = (ts) => msg(ts, `<@${BOT_USER}> please deploy`);

test('scan separates triggers from replies', () => {
  const stats = scan([mention(100), msg(110, 'unrelated chatter'), msg(120, 'done', true)],
    BOT, BOT_USER);
  assert.equal(stats.triggers, 1);
  assert.equal(stats.replies, 1);
  assert.equal(stats.unanswered, 0);
});

test('the bots own mention of itself is not a trigger', () => {
  const messages = [msg(100, `<@${BOT_USER}> was asked`, true)];
  assert.equal(scan(messages, BOT, BOT_USER).triggers, 0);
});

test('a run of mentions after the last reply is the finding', () => {
  const messages = [msg(1000, 'on it', true), mention(5000), mention(9000), mention(13000)];
  const [state, detail] = verdict(scan(messages, BOT, BOT_USER));
  assert.equal(state, 'silent');
  assert.match(detail, /3 mention\(s\)/);
});

test('an app that never replied is a different diagnosis', () => {
  const messages = [mention(1000), mention(2000), mention(3000), mention(4000)];
  const [state, detail] = verdict(scan(messages, BOT, BOT_USER));
  assert.equal(state, 'never-answered');
  assert.match(detail, /never configured/);
});

test('a reply after the last mention is healthy', () => {
  const messages = [mention(1000), msg(1100, 'done', true)];
  assert.equal(verdict(scan(messages, BOT, BOT_USER))[0], 'answering');
});

test('one unanswered mention is not enough', () => {
  const messages = [msg(1000, 'done', true), mention(2000)];
  assert.equal(verdict(scan(messages, BOT, BOT_USER))[0], 'too-little-evidence');
});

test('a quiet channel is not evidence', () => {
  const messages = [msg(1000, 'morning'), msg(2000, 'morning')];
  assert.equal(verdict(scan(messages, BOT, BOT_USER))[0], 'no-triggers');
});

test('the threshold is adjustable', () => {
  const messages = [msg(1000, 'done', true), mention(2000), mention(3000)];
  assert.equal(verdict(scan(messages, BOT, BOT_USER), 2)[0], 'silent');
});