your sitemap lists URLs that redirect, 404 or say noindex

"""Remove URLs from a sitemap that contradict it: 404s, redirects, noindex.

A sitemap says "these pages are worth indexing". Listing a page that 404s or carries
a noindex says the opposite in the same breath, and Search Console reports it.
"""
import argparse
import logging
import re
import sys
import xml.etree.ElementTree as ET
from urllib.parse import urlsplit

import requests

logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s")
log = logging.getLogger("sitemap_prune")

NS = {"sm": "http://www.sitemaps.org/schemas/sitemap/0.9"}
MAX_URLS = 50_000          # protocol limit
MAX_BYTES = 50 * 1024 ** 2  # 50 MB uncompressed


def verdict(url, sitemap_host, status, final_url, head_noindex, meta_noindex, canonical):
    """Pure decision function: why should this URL not be in the sitemap?

    Returns a list of reasons; empty means keep. Kept separate from the fetching so
    the rules can be read and tested without a network.
    """
    reasons = []
    if status >= 400:
        reasons.append(f"returns {status}")
    elif status >= 300 or (final_url and final_url != url):
        reasons.append(f"redirects to {final_url or 'elsewhere'}; list the destination")
    if head_noindex:
        reasons.append("X-Robots-Tag: noindex")
    if meta_noindex:
        reasons.append("meta robots noindex")
    if canonical and canonical.rstrip("/") != url.rstrip("/"):
        reasons.append(f"canonical points to {canonical}")
    if urlsplit(url).netloc != sitemap_host:
        reasons.append(f"host {urlsplit(url).netloc} is not the sitemap's host "
                       f"{sitemap_host}; cross-submission needs robots.txt")
    return reasons


def parse_urls(xml_text):
    """Return (child_sitemaps, urls). One level of index expansion is the protocol max."""
    root = ET.fromstring(xml_text)
    children = [e.text.strip() for e in root.findall(".//sm:sitemap/sm:loc", NS) if e.text]
    urls = [e.text.strip() for e in root.findall(".//sm:url/sm:loc", NS) if e.text]
    return children, urls


def inspect(session, url):
    r = session.get(url, timeout=30, allow_redirects=True)
    body = r.text[:200_000]
    head_noindex = "noindex" in (r.headers.get("X-Robots-Tag", "").lower())
    m = re.search(r'<meta[^>]+name=["\']robots["\'][^>]*>', body, re.I)
    meta_noindex = bool(m and "noindex" in m.group(0).lower())
    c = re.search(r'<link[^>]+rel=["\']canonical["\'][^>]*href=["\']([^"\']+)', body, re.I)
    status = r.history[0].status_code if r.history else r.status_code
    return status, r.url, head_noindex, meta_noindex, (c.group(1) if c else None)


def main():
    ap = argparse.ArgumentParser()
    ap.add_argument("--sitemap", required=True)
    ap.add_argument("--out", help="write the cleaned sitemap here")
    args = ap.parse_args()

    s = requests.Session()
    s.headers.update({"User-Agent": "sitemap-prune/1.0"})

    top = s.get(args.sitemap, timeout=30)
    top.raise_for_status()
    if len(top.content) > MAX_BYTES:
        log.warning("sitemap is over the 50 MB uncompressed limit")
    children, urls = parse_urls(top.text)
    for child in children:
        r = s.get(child, timeout=30)
        urls.extend(parse_urls(r.text)[1])
    if len(urls) > MAX_URLS:
        log.warning("%d URLs -- over the %d limit; split into an index", len(urls), MAX_URLS)

    host = urlsplit(args.sitemap).netloc
    keep, drop = [], []
    for u in urls:
        try:
            reasons = verdict(u, host, *inspect(s, u))
        except requests.RequestException as e:
            reasons = [f"request failed: {e.__class__.__name__}"]
        if reasons:
            drop.append((u, reasons))
            log.warning("DROP %s -- %s", u, "; ".join(reasons))
        else:
            keep.append(u)

    log.info("%d keep, %d to remove", len(keep), len(drop))
    if args.out and args.out != "/dev/null":
        body = "\n".join(f"  <url>\n    <loc>{u}</loc>\n  </url>" for u in keep)
        with open(args.out, "w", encoding="utf-8") as fh:
            fh.write('<?xml version="1.0" encoding="UTF-8"?>\n'
                     '<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">\n'
                     f"{body}\n</urlset>\n")
        log.info("wrote %s -- compare it against the original before replacing", args.out)
    return 1 if drop else 0


if __name__ == "__main__":
    sys.exit(main())

/**
 * Remove URLs from a sitemap that contradict it: 404s, redirects, noindex.
 *
 * A sitemap says "these pages are worth indexing". Listing a page that 404s or
 * carries a noindex says the opposite in the same breath.
 */
import { writeFileSync } from 'node:fs';

const MAX_URLS = 50_000;                 // protocol limit
const MAX_BYTES = 50 * 1024 ** 2;        // 50 MB uncompressed

/**
 * Pure decision function: why should this URL not be in the sitemap?
 * Empty array means keep. Separate from fetching so the rules can be tested.
 */
export function verdict({ url, sitemapHost, status, finalUrl, headNoindex, metaNoindex, canonical }) {
  const reasons = [];
  if (status >= 400) reasons.push(`returns ${status}`);
  else if (status >= 300 || (finalUrl && finalUrl !== url)) {
    reasons.push(`redirects to ${finalUrl ?? 'elsewhere'}; list the destination`);
  }
  if (headNoindex) reasons.push('X-Robots-Tag: noindex');
  if (metaNoindex) reasons.push('meta robots noindex');
  if (canonical && canonical.replace(/\/$/, '') !== url.replace(/\/$/, '')) {
    reasons.push(`canonical points to ${canonical}`);
  }
  const host = new URL(url).host;
  if (host !== sitemapHost) {
    reasons.push(`host ${host} is not the sitemap's host ${sitemapHost}; `
      + 'cross-submission needs robots.txt');
  }
  return reasons;
}

/** Return { children, urls }. One level of index expansion is the protocol max. */
export function parseUrls(xml) {
  const locs = (block) => [...block.matchAll(/<loc>\s*([^<\s]+)\s*<\/loc>/g)].map((m) => m[1]);
  const sitemapBlocks = xml.match(/<sitemap>[\s\S]*?<\/sitemap>/g) ?? [];
  const urlBlocks = xml.match(/<url>[\s\S]*?<\/url>/g) ?? [];
  return { children: sitemapBlocks.flatMap(locs), urls: urlBlocks.flatMap(locs) };
}

async function inspect(url) {
  const r = await fetch(url, { redirect: 'follow' });
  const body = (await r.text()).slice(0, 200_000);
  const headNoindex = (r.headers.get('x-robots-tag') ?? '').toLowerCase().includes('noindex');
  const meta = body.match(/<meta[^>]+name=["']robots["'][^>]*>/i);
  const canon = body.match(/<link[^>]+rel=["']canonical["'][^>]*href=["']([^"']+)/i);
  return {
    status: r.redirected ? 301 : r.status,
    finalUrl: r.url,
    headNoindex,
    metaNoindex: Boolean(meta && meta[0].toLowerCase().includes('noindex')),
    canonical: canon?.[1] ?? null,
  };
}

async function main() {
  const arg = (n) => process.argv[process.argv.indexOf(n) + 1];
  const sitemap = arg('--sitemap');
  const out = process.argv.includes('--out') ? arg('--out') : null;

  const top = await fetch(sitemap);
  const xml = await top.text();
  if (Buffer.byteLength(xml) > MAX_BYTES) console.warn('sitemap is over the 50 MB limit');
  const { children, urls } = parseUrls(xml);
  for (const child of children) urls.push(...parseUrls(await (await fetch(child)).text()).urls);
  if (urls.length > MAX_URLS) console.warn(`${urls.length} URLs -- over the ${MAX_URLS} limit`);

  const sitemapHost = new URL(sitemap).host;
  const keep = []; const drop = [];
  for (const url of urls) {
    let reasons;
    try { reasons = verdict({ url, sitemapHost, ...(await inspect(url)) }); }
    catch (e) { reasons = [`request failed: ${e.name}`]; }
    if (reasons.length) { drop.push([url, reasons]); console.warn(`DROP ${url} -- ${reasons.join('; ')}`); }
    else keep.push(url);
  }

  console.log(`${keep.length} keep, ${drop.length} to remove`);
  if (out && out !== '/dev/null') {
    const body = keep.map((u) => `  <url>\n    <loc>${u}</loc>\n  </url>`).join('\n');
    writeFileSync(out, '<?xml version="1.0" encoding="UTF-8"?>\n'
      + '<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">\n'
      + `${body}\n</urlset>\n`);
    console.log(`wrote ${out} -- compare it against the original before replacing`);
  }
  process.exit(drop.length ? 1 : 0);
}

if (import.meta.url === `file://${process.argv[1]}`) main();

from sitemap_prune import verdict, parse_urls

HOST = "example.com"
SM = '<?xml version="1.0"?><urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">' \
     "<url><loc>https://example.com/a</loc></url></urlset>"


def ok(url="https://example.com/a", **kw):
    args = dict(status=200, final_url=url, head_noindex=False,
                meta_noindex=False, canonical=url)
    args.update(kw)
    return verdict(url, HOST, args["status"], args["final_url"], args["head_noindex"],
                   args["meta_noindex"], args["canonical"])


def test_a_healthy_url_is_kept():
    assert ok() == []


def test_a_404_is_dropped():
    assert any("returns 404" in r for r in ok(status=404))


def test_a_redirect_is_dropped_with_the_destination():
    r = ok(status=301, final_url="https://example.com/b")
    assert any("redirects to https://example.com/b" in x for x in r)


def test_meta_noindex_is_dropped():
    assert any("meta robots noindex" in r for r in ok(meta_noindex=True))


def test_x_robots_tag_noindex_is_dropped():
    assert any("X-Robots-Tag" in r for r in ok(head_noindex=True))


def test_a_trailing_slash_is_not_a_canonical_mismatch():
    """Same page. Treating this as a mismatch deletes good URLs."""
    assert ok(url="https://example.com/a/", canonical="https://example.com/a") == []


def test_a_real_canonical_mismatch_is_dropped():
    assert any("canonical points to" in r
               for r in ok(canonical="https://example.com/other"))


def test_a_foreign_host_is_dropped():
    assert any("is not the sitemap's host" in r
               for r in ok(url="https://cdn.example.net/a"))


def test_parse_urls_reads_locs():
    assert parse_urls(SM)[1] == ["https://example.com/a"]

import { test } from 'node:test';
import assert from 'node:assert/strict';
import { verdict, parseUrls } from './sitemap-prune.mjs';

const ok = (over = {}) => {
  const url = over.url ?? 'https://example.com/a';
  return verdict({
    url, sitemapHost: 'example.com', status: 200, finalUrl: url,
    headNoindex: false, metaNoindex: false, canonical: url, ...over,
  });
};

test('a healthy URL is kept', () => assert.deepEqual(ok(), []));

test('a 404 is dropped', () => {
  assert.ok(ok({ status: 404 }).some((r) => r.includes('returns 404')));
});

test('a redirect is dropped with the destination', () => {
  assert.ok(ok({ status: 301, finalUrl: 'https://example.com/b' })
    .some((r) => r.includes('redirects to https://example.com/b')));
});

test('a trailing slash is not a canonical mismatch', () => {
  assert.deepEqual(ok({ url: 'https://example.com/a/', canonical: 'https://example.com/a' }), []);
});

test('a foreign host is dropped', () => {
  assert.ok(ok({ url: 'https://cdn.example.net/a' })
    .some((r) => r.includes("is not the sitemap's host")));
});

test('parseUrls reads locs', () => {
  const xml = '<urlset><url><loc>https://example.com/a</loc></url></urlset>';
  assert.deepEqual(parseUrls(xml).urls, ['https://example.com/a']);
});