CloudWatch log groups keep everything forever by default

"""Find CloudWatch log groups with no retention policy and set one.

New log groups default to Never expire, and they are created for you by Lambda,
ECS and API Gateway rather than by anyone deciding to keep logs forever. Setting
retention applies to existing data, so one pass can free years of storage.
"""
import argparse
import logging
import sys

import boto3

logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s")
log = logging.getLogger("cloudwatch_retention_audit")

STORAGE_PER_GB_MONTH = 0.03
# CloudWatch accepts only this set; anything else is rejected at the API.
VALID_RETENTION_DAYS = {1, 3, 5, 7, 14, 30, 60, 90, 120, 150, 180, 365, 400,
                        545, 731, 1096, 1827, 2192, 2557, 2922, 3288, 3653}


def assess(group):
    """Pure decision function over one describe-log-groups entry."""
    stored = group.get("storedBytes", 0)
    gb = stored / 1_073_741_824
    return {
        "unbounded": "retentionInDays" not in group,
        "gb": gb,
        "monthly_usd": gb * STORAGE_PER_GB_MONTH,
        "orphan_hint": group.get("logGroupName", "").startswith("/aws/lambda/"),
    }


def main():
    ap = argparse.ArgumentParser()
    ap.add_argument("--region", default="us-east-1")
    ap.add_argument("--prefix", default="", help="only groups starting with this")
    ap.add_argument("--set-days", type=int, help="retention to apply")
    ap.add_argument("--apply", action="store_true")
    args = ap.parse_args()

    if args.set_days and args.set_days not in VALID_RETENTION_DAYS:
        log.error("%d is not an accepted retention; choose one of %s",
                  args.set_days, sorted(VALID_RETENTION_DAYS))
        return 2

    logs = boto3.client("logs", region_name=args.region)
    paginator = logs.get_paginator("describe_log_groups")
    kwargs = {"logGroupNamePrefix": args.prefix} if args.prefix else {}

    unbounded, total = [], 0.0
    for page in paginator.paginate(**kwargs):
        for g in page["logGroups"]:
            info = assess(g)
            if info["unbounded"]:
                unbounded.append((g["logGroupName"], info))
                total += info["monthly_usd"]

    for name, info in sorted(unbounded, key=lambda x: -x[1]["gb"])[:40]:
        log.warning("NO RETENTION  %7.2f GB  $%5.2f/mo  %s", info["gb"],
                    info["monthly_usd"], name)
    if unbounded:
        log.warning("%d group(s) keep logs forever, about $%.2f/month in storage",
                    len(unbounded), total)
    else:
        log.info("every log group has a retention policy")

    if args.set_days and args.apply:
        for name, _ in unbounded:
            logs.put_retention_policy(logGroupName=name, retentionInDays=args.set_days)
        log.info("set %d-day retention on %d group(s)", args.set_days, len(unbounded))
    elif args.set_days:
        log.info("WOULD set %d-day retention on %d group(s) -- pass --apply",
                 args.set_days, len(unbounded))
    return 0


if __name__ == "__main__":
    sys.exit(main())

/**
 * Find CloudWatch log groups with no retention policy and set one.
 *
 * New log groups default to Never expire, and they are created for you by Lambda,
 * ECS and API Gateway rather than by anyone deciding to keep logs forever.
 */
import {
  CloudWatchLogsClient,
  DescribeLogGroupsCommand,
  PutRetentionPolicyCommand,
} from '@aws-sdk/client-cloudwatch-logs';

const STORAGE_PER_GB_MONTH = 0.03;
// CloudWatch accepts only this set; anything else is rejected at the API.
export const VALID_RETENTION_DAYS = new Set([1, 3, 5, 7, 14, 30, 60, 90, 120, 150, 180,
  365, 400, 545, 731, 1096, 1827, 2192, 2557, 2922, 3288, 3653]);

/** Pure decision function over one describe-log-groups entry. */
export function assess(group) {
  const gb = (group.storedBytes ?? 0) / 1_073_741_824;
  return {
    unbounded: group.retentionInDays === undefined,
    gb,
    monthlyUsd: gb * STORAGE_PER_GB_MONTH,
    orphanHint: (group.logGroupName ?? '').startsWith('/aws/lambda/'),
  };
}

async function main() {
  const region = process.env.AWS_REGION ?? 'us-east-1';
  const apply = process.argv.includes('--apply');
  const setDays = Number(process.argv[process.argv.indexOf('--set-days') + 1]);
  if (process.argv.includes('--set-days') && !VALID_RETENTION_DAYS.has(setDays)) {
    console.error(`${setDays} is not an accepted retention value`);
    process.exit(2);
  }
  const client = new CloudWatchLogsClient({ region });

  const unbounded = [];
  let total = 0;
  let nextToken;
  do {
    const page = await client.send(new DescribeLogGroupsCommand({ nextToken }));
    for (const g of page.logGroups ?? []) {
      const info = assess(g);
      if (info.unbounded) { unbounded.push([g.logGroupName, info]); total += info.monthlyUsd; }
    }
    nextToken = page.nextToken;
  } while (nextToken);

  for (const [name, info] of unbounded.sort((a, b) => b[1].gb - a[1].gb).slice(0, 40)) {
    console.warn(`NO RETENTION  ${info.gb.toFixed(2)} GB  $${info.monthlyUsd.toFixed(2)}/mo  ${name}`);
  }
  if (unbounded.length) {
    console.warn(`${unbounded.length} group(s) keep logs forever, about $${total.toFixed(2)}/month`);
  }

  if (process.argv.includes('--set-days')) {
    if (apply) {
      for (const [name] of unbounded) {
        await client.send(new PutRetentionPolicyCommand({
          logGroupName: name, retentionInDays: setDays }));
      }
      console.log(`set ${setDays}-day retention on ${unbounded.length} group(s)`);
    } else {
      console.log(`WOULD set ${setDays}-day retention on ${unbounded.length} group(s) -- pass --apply`);
    }
  }
}

if (import.meta.url === `file://${process.argv[1]}`) main();

from cloudwatch_retention_audit import assess, VALID_RETENTION_DAYS


def test_missing_key_means_unbounded():
    assert assess({"logGroupName": "/aws/lambda/x", "storedBytes": 0})["unbounded"] is True


def test_a_retention_of_any_value_is_bounded():
    g = {"logGroupName": "/x", "storedBytes": 0, "retentionInDays": 7}
    assert assess(g)["unbounded"] is False


def test_cost_scales_with_stored_bytes():
    ten_gb = 10 * 1_073_741_824
    assert round(assess({"logGroupName": "/x", "storedBytes": ten_gb})["monthly_usd"], 2) == 0.30


def test_lambda_groups_are_hinted_as_likely_orphans():
    assert assess({"logGroupName": "/aws/lambda/gone", "storedBytes": 1})["orphan_hint"]


def test_45_days_is_not_an_accepted_retention():
    """Looks reasonable, rejected by the API. Worth failing early on."""
    assert 45 not in VALID_RETENTION_DAYS
    assert 30 in VALID_RETENTION_DAYS and 60 in VALID_RETENTION_DAYS

import { test } from 'node:test';
import assert from 'node:assert/strict';
import { assess, VALID_RETENTION_DAYS } from './cloudwatch-retention-audit.mjs';

test('a missing key means unbounded', () => {
  assert.equal(assess({ logGroupName: '/aws/lambda/x', storedBytes: 0 }).unbounded, true);
});

test('any retention value means bounded', () => {
  assert.equal(assess({ logGroupName: '/x', storedBytes: 0, retentionInDays: 7 }).unbounded, false);
});

test('cost scales with stored bytes', () => {
  const tenGb = 10 * 1_073_741_824;
  assert.equal(assess({ logGroupName: '/x', storedBytes: tenGb }).monthlyUsd.toFixed(2), '0.30');
});

test('45 days is not an accepted retention', () => {
  assert.equal(VALID_RETENTION_DAYS.has(45), false);
  assert.ok(VALID_RETENTION_DAYS.has(30) && VALID_RETENTION_DAYS.has(60));
});