a canonical tag pointing at staging, a redirect or nothing

"""Audit and repair rel=canonical tags in built HTML.

Canonicals are generated from one base URL, so when the base is wrong every page is
wrong at once -- and the page looks entirely normal in a browser, because a canonical
is invisible.
"""
import argparse
import logging
import re
import sys
from pathlib import Path
from urllib.parse import urlsplit, urlunsplit

logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s")
log = logging.getLogger("canonical_audit")

CANON = re.compile(r'<link[^>]+rel=["\']canonical["\'][^>]*>', re.I)
HREF = re.compile(r'href=["\']([^"\']+)["\']', re.I)


def classify(canonicals, page_url, live_origin):
    """Pure decision function over the canonical tags found on one page.

    canonicals: list of href strings, in document order.
    Returns a list of problems; empty means the page is fine.
    """
    problems = []
    if not canonicals:
        return ["no canonical tag"]
    if len(canonicals) > 1:
        # Worse than none: Google ignores all of them when there is more than one.
        problems.append(f"{len(canonicals)} canonical tags -- Google ignores all of them")
    href = canonicals[0]
    if not href.startswith(("http://", "https://")):
        problems.append(f"relative canonical {href!r} -- resolves against the page URL, "
                        "which surprises on parameterised URLs")
        return problems
    parts = urlsplit(href)
    origin = urlunsplit((parts.scheme, parts.netloc, "", "", ""))
    if live_origin and origin != live_origin.rstrip("/"):
        problems.append(f"origin {origin} is not the live origin {live_origin}")
    if parts.scheme == "http":
        problems.append("canonical uses http; it should match the served scheme")
    # A path repeated back to back is the classic base-plus-prefix concatenation bug.
    segs = [s for s in parts.path.split("/") if s]
    for i in range(len(segs) - 1):
        if segs[i] and segs[i] == segs[i + 1]:
            problems.append(f"path segment {segs[i]!r} appears twice -- a base URL and a "
                            "path prefix were probably concatenated")
            break
    if page_url and href.rstrip("/") != page_url.rstrip("/") and not problems:
        problems.append(f"canonical {href} differs from the page URL {page_url}; "
                        "correct for a duplicate, wrong for a page you want indexed")
    return problems


def swap_origin(html, old_origin, new_origin):
    """Replace only the scheme-and-host portion of canonical hrefs.

    Narrow on purpose. A greedy origin rewrite is how a good URL becomes a bare
    origin and a sitemap reference becomes nonsense.
    """
    def fix(m):
        return m.group(0).replace(old_origin.rstrip("/"), new_origin.rstrip("/"))
    return CANON.sub(fix, html)


def hrefs_in(html):
    return [HREF.search(tag).group(1) for tag in CANON.findall(html) if HREF.search(tag)]


def main():
    ap = argparse.ArgumentParser()
    ap.add_argument("--dir", required=True, help="directory of built HTML")
    ap.add_argument("--live-origin", help="https://example.com")
    ap.add_argument("--from-origin", help="origin to replace when using --apply")
    ap.add_argument("--apply", action="store_true")
    args = ap.parse_args()

    files = sorted(Path(args.dir).rglob("*.html"))
    log.info("%d file(s)", len(files))
    bad = 0
    for f in files:
        html = f.read_text(encoding="utf-8", errors="replace")
        found = hrefs_in(html)
        problems = classify(found, None, args.live_origin)
        if problems:
            bad += 1
            log.warning("%s -- %s", f, "; ".join(problems))
        if args.apply and args.from_origin and args.live_origin:
            fixed = swap_origin(html, args.from_origin, args.live_origin)
            if fixed != html:
                f.write_text(fixed, encoding="utf-8")
                log.info("rewrote canonical origin in %s", f)
    if not args.apply and args.from_origin:
        log.info("WOULD rewrite origins -- pass --apply")
    log.info("%d file(s) with problems", bad)
    return 1 if bad else 0


if __name__ == "__main__":
    sys.exit(main())

/**
 * Audit and repair rel=canonical tags in built HTML.
 *
 * Canonicals are generated from one base URL, so when the base is wrong every page
 * is wrong at once -- and looks entirely normal in a browser.
 */
import { readdirSync, readFileSync, writeFileSync, statSync } from 'node:fs';
import { join } from 'node:path';

const CANON = /<link[^>]+rel=["']canonical["'][^>]*>/gi;
const HREF = /href=["']([^"']+)["']/i;

/**
 * Pure decision function over the canonical tags found on one page.
 * Returns a list of problems; empty means fine.
 */
export function classify(canonicals, pageUrl, liveOrigin) {
  const problems = [];
  if (!canonicals.length) return ['no canonical tag'];
  if (canonicals.length > 1) {
    // Worse than none: Google ignores all of them when there is more than one.
    problems.push(`${canonicals.length} canonical tags -- Google ignores all of them`);
  }
  const href = canonicals[0];
  if (!/^https?:\/\//.test(href)) {
    problems.push(`relative canonical "${href}" -- resolves against the page URL, `
      + 'which surprises on parameterised URLs');
    return problems;
  }
  const u = new URL(href);
  const origin = `${u.protocol}//${u.host}`;
  if (liveOrigin && origin !== liveOrigin.replace(/\/$/, '')) {
    problems.push(`origin ${origin} is not the live origin ${liveOrigin}`);
  }
  if (u.protocol === 'http:') problems.push('canonical uses http; it should match the served scheme');
  const segs = u.pathname.split('/').filter(Boolean);
  for (let i = 0; i < segs.length - 1; i += 1) {
    if (segs[i] && segs[i] === segs[i + 1]) {
      problems.push(`path segment "${segs[i]}" appears twice -- a base URL and a path `
        + 'prefix were probably concatenated');
      break;
    }
  }
  if (pageUrl && href.replace(/\/$/, '') !== pageUrl.replace(/\/$/, '') && !problems.length) {
    problems.push(`canonical ${href} differs from the page URL ${pageUrl}; `
      + 'correct for a duplicate, wrong for a page you want indexed');
  }
  return problems;
}

/**
 * Replace only the scheme-and-host portion of canonical hrefs. Narrow on purpose:
 * a greedy origin rewrite is how a good URL becomes a bare origin.
 */
export function swapOrigin(html, oldOrigin, newOrigin) {
  return html.replace(CANON, (tag) =>
    tag.replaceAll(oldOrigin.replace(/\/$/, ''), newOrigin.replace(/\/$/, '')));
}

export const hrefsIn = (html) =>
  (html.match(CANON) ?? []).map((t) => t.match(HREF)?.[1]).filter(Boolean);

const walk = (dir) => readdirSync(dir).flatMap((n) => {
  const p = join(dir, n);
  return statSync(p).isDirectory() ? walk(p) : (p.endsWith('.html') ? [p] : []);
});

async function main() {
  const arg = (n) => process.argv[process.argv.indexOf(n) + 1];
  const dir = arg('--dir');
  const liveOrigin = process.argv.includes('--live-origin') ? arg('--live-origin') : null;
  const fromOrigin = process.argv.includes('--from-origin') ? arg('--from-origin') : null;
  const apply = process.argv.includes('--apply');

  const files = walk(dir).sort();
  console.log(`${files.length} file(s)`);
  let bad = 0;
  for (const f of files) {
    const html = readFileSync(f, 'utf8');
    const problems = classify(hrefsIn(html), null, liveOrigin);
    if (problems.length) { bad += 1; console.warn(`${f} -- ${problems.join('; ')}`); }
    if (apply && fromOrigin && liveOrigin) {
      const fixed = swapOrigin(html, fromOrigin, liveOrigin);
      if (fixed !== html) { writeFileSync(f, fixed); console.log(`rewrote canonical origin in ${f}`); }
    }
  }
  if (!apply && fromOrigin) console.log('WOULD rewrite origins -- pass --apply');
  console.log(`${bad} file(s) with problems`);
  process.exit(bad ? 1 : 0);
}

if (import.meta.url === `file://${process.argv[1]}`) main();

from canonical_audit import classify, hrefs_in, swap_origin

LIVE = "https://example.com"
TAG = '<link rel="canonical" href="{}">'


def test_a_single_correct_canonical_is_clean():
    assert classify(["https://example.com/a/"], None, LIVE) == []


def test_no_canonical_is_reported():
    assert classify([], None, LIVE) == ["no canonical tag"]


def test_two_canonicals_are_worse_than_none():
    """Google ignores all of them when there is more than one."""
    p = classify(["https://example.com/a/", "https://example.com/b/"], None, LIVE)
    assert any("ignores all of them" in x for x in p)


def test_a_staging_origin_is_reported():
    p = classify(["https://staging.example.net/a/"], None, LIVE)
    assert any("is not the live origin" in x for x in p)


def test_a_doubled_path_segment_is_reported():
    p = classify(["https://example.com/blog/blog/post/"], None, LIVE)
    assert any("appears twice" in x for x in p)


def test_a_relative_canonical_is_reported():
    p = classify(["/a/"], None, LIVE)
    assert any("relative canonical" in x for x in p)


def test_swap_origin_changes_the_host_and_keeps_the_path():
    """The failure mode is a greedy replace that eats the path."""
    html = TAG.format("http://localhost:4321/blog/post/")
    out = swap_origin(html, "http://localhost:4321", LIVE)
    assert out == TAG.format("https://example.com/blog/post/")


def test_swap_origin_leaves_other_links_alone():
    html = ('<a href="http://localhost:4321/x">x</a>'
            + TAG.format("http://localhost:4321/y"))
    out = swap_origin(html, "http://localhost:4321", LIVE)
    assert 'href="http://localhost:4321/x"' in out
    assert 'href="https://example.com/y"' in out


def test_hrefs_in_finds_the_tag():
    assert hrefs_in(TAG.format("https://example.com/a/")) == ["https://example.com/a/"]

import { test } from 'node:test';
import assert from 'node:assert/strict';
import { classify, hrefsIn, swapOrigin } from './canonical-audit.mjs';

const LIVE = 'https://example.com';
const tag = (h) => `<link rel="canonical" href="${h}">`;

test('a single correct canonical is clean', () => {
  assert.deepEqual(classify(['https://example.com/a/'], null, LIVE), []);
});

test('no canonical is reported', () => {
  assert.deepEqual(classify([], null, LIVE), ['no canonical tag']);
});

test('two canonicals are worse than none', () => {
  const p = classify(['https://example.com/a/', 'https://example.com/b/'], null, LIVE);
  assert.ok(p.some((x) => x.includes('ignores all of them')));
});

test('a doubled path segment is reported', () => {
  const p = classify(['https://example.com/blog/blog/post/'], null, LIVE);
  assert.ok(p.some((x) => x.includes('appears twice')));
});

test('swapOrigin changes the host and keeps the path', () => {
  const out = swapOrigin(tag('http://localhost:4321/blog/post/'), 'http://localhost:4321', LIVE);
  assert.equal(out, tag('https://example.com/blog/post/'));
});

test('swapOrigin leaves other links alone', () => {
  const html = `<a href="http://localhost:4321/x">x</a>${tag('http://localhost:4321/y')}`;
  const out = swapOrigin(html, 'http://localhost:4321', LIVE);
  assert.ok(out.includes('href="http://localhost:4321/x"'));
  assert.ok(out.includes('href="https://example.com/y"'));
});

test('hrefsIn finds the tag', () => {
  assert.deepEqual(hrefsIn(tag('https://example.com/a/')), ['https://example.com/a/']);
});