Nested GraphQL connections truncate at 100 per parent

"""Find nested GraphQL connections that truncated once per parent.

Read only, and queries only. GitHub's GraphQL endpoint takes a document in the
request body, so a read is carried by POST there just as a write would be; that
is transport, not intent. This script sends queries and refuses any document
containing a mutation or a subscription before it opens a socket. Nothing is
written and the repair is printed rather than performed.

Every connection carries its own cursor. Paginating the outer connection does
nothing for the inner ones: each new outer page restarts them from the start, so
each parent returns its first n children and stops, with no error and no marker.
The evidence is totalCount sitting next to a shorter list of nodes, once per
parent, and pageInfo.hasNextPage coming back true on a connection nobody
intended to follow.

What this can and cannot see: the API cannot tell whether your client follows a
cursor. What it can do is measure one response, report every connection that
returned fewer items than it holds, name the connections that asked for neither
totalCount nor pageInfo and therefore cannot be checked at all, and count the
follow-up queries that doing it properly would cost.

Environment:

    GITHUB_TOKEN    a token with read access to the GraphQL API
"""
import argparse
import json
import logging
import os
import sys
from pathlib import Path

import requests

logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s")
log = logging.getLogger("github_graphql_nested")

API = "https://api.github.com"
UA = "github-graphql-nested/1.0"

POINTS_PER_QUERY = 1

# Deliberately imperfect: the inner connection asks for totalCount but not for
# pageInfo, which is the common shape. It proves the truncation and gives you no
# cursor to continue from.
DEFAULT_QUERY = (
    "query($login: String!, $outer: Int = 5, $inner: Int = 5) {"
    " repositoryOwner(login: $login) {"
    " repositories(first: $outer, orderBy: {field: PUSHED_AT, direction: DESC}) {"
    " totalCount pageInfo { hasNextPage endCursor }"
    " nodes { name issues(first: $inner, states: OPEN) {"
    " totalCount nodes { number } } }"
    " } } }"
)


def strip_noise(document):
    """Remove GraphQL comments and string literals from a document. Pure."""
    src = str(document or "")
    out = []
    i, n = 0, len(src)
    while i < n:
        ch = src[i]
        if ch == "#":
            while i < n and src[i] != "\n":
                i += 1
            continue
        if src.startswith('"""', i):
            j = src.find('"""', i + 3)
            i = n if j < 0 else j + 3
            out.append(" ")
            continue
        if ch == '"':
            i += 1
            while i < n and src[i] != '"':
                i += 2 if src[i] == "\\" else 1
            i += 1
            out.append(" ")
            continue
        out.append(ch)
        i += 1
    return "".join(out)


def operations(document):
    """The top-level operations in a document, in order. Pure."""
    src = strip_noise(document)
    ops, depth, word, declared = [], 0, "", None
    for ch in src + " ":
        if ch.isalnum() or ch == "_":
            word += ch
            continue
        if word:
            if depth == 0 and word in ("query", "mutation", "subscription", "fragment"):
                declared = word
            word = ""
        if ch == "{":
            if depth == 0:
                ops.append(declared or "query")
                declared = None
            depth += 1
        elif ch == "}":
            depth = max(0, depth - 1)
    return ops


def refusal(document):
    """Why this document will not be sent, or None if it is a read. Pure."""
    ops = operations(document)
    if not ops:
        return "the document contains no operation to send."
    for kind in ("mutation", "subscription"):
        if kind in ops:
            return ("the document contains a %s. This script sends queries only: "
                    "a query is a read, and the section it belongs to promises "
                    "its scripts never write." % kind)
    return None


def outer_text(block):
    """A selection set with everything nested inside it blanked out. Pure.

    So that a pageInfo belonging to an inner connection is never credited to the
    connection that contains it, which is the mistake that makes a text-level
    audit of nested queries useless.
    """
    out, depth = [], 0
    for ch in str(block or ""):
        if ch == "{":
            depth += 1
            out.append(" ")
            continue
        if ch == "}":
            depth = max(0, depth - 1)
            out.append(" ")
            continue
        out.append(ch if depth == 0 else " ")
    return "".join(out)


def connection_fields(document, _depth=0, _stripped=False):
    """Every connection in the document, found by shape. Pure.

    A connection is a field whose own selection set contains nodes or edges.
    Identifying them that way rather than by their slicing argument means a
    connection paginated by a variable, or one written without arguments at all,
    is still found.
    """
    src = str(document or "") if _stripped else strip_noise(document)
    out, i, n, word, field = [], 0, len(src), "", ""
    while i < n:
        ch = src[i]
        if ch.isalnum() or ch == "_":
            word += ch
            i += 1
            continue
        # The field name has to survive the whitespace and the argument list
        # between it and its selection set, so it is remembered rather than
        # read off whatever happens to precede the brace.
        if word:
            field, word = word, ""
        if ch == "(":
            j, level = i, 0
            while j < n:
                if src[j] == "(":
                    level += 1
                elif src[j] == ")":
                    level -= 1
                    if level == 0:
                        break
                j += 1
            i = j + 1
            continue
        if ch == "{":
            j, level = i, 0
            while j < n:
                if src[j] == "{":
                    level += 1
                elif src[j] == "}":
                    level -= 1
                    if level == 0:
                        break
                j += 1
            block = src[i + 1:j]
            own = outer_text(block).split()
            if field and ("nodes" in own or "edges" in own):
                out.append({"field": field, "depth": _depth,
                            "has_page_info": "pageInfo" in own,
                            "has_total_count": "totalCount" in own})
                out.extend(connection_fields(block, _depth + 1, True))
            else:
                out.extend(connection_fields(block, _depth, True))
            i, field = j + 1, ""
            continue
        if ch == "}":
            field = ""
        i += 1
    return out


def unauditable(fields):
    """Inner connections that asked for neither totalCount nor pageInfo. Pure."""
    return [f for f in fields
            if f["depth"] >= 1 and not f["has_total_count"] and not f["has_page_info"]]


def unresumable(fields):
    """Inner connections that can be seen to truncate but not continued. Pure."""
    return [f for f in fields
            if f["depth"] >= 1 and f["has_total_count"] and not f["has_page_info"]]


def is_connection(value):
    """Whether a decoded object is a connection. Pure."""
    if not isinstance(value, dict):
        return False
    return isinstance(value.get("nodes"), list) or isinstance(value.get("edges"), list)


def walk_connections(data, path="", depth=0):
    """Every connection in a decoded response, with its path and depth. Pure.

    Depth counts connections above this one rather than keys, so an inner
    connection is depth 1 however many plain objects sit between it and its
    parent connection.
    """
    out = []
    if isinstance(data, dict):
        if is_connection(data):
            items = data.get("nodes")
            if not isinstance(items, list):
                items = data.get("edges") or []
            page = data.get("pageInfo")
            total = data.get("totalCount")
            out.append({
                "path": path or "(root)",
                "depth": depth,
                "returned": len(items),
                "total_count": total if isinstance(total, int) else None,
                "has_next_page": page.get("hasNextPage") if isinstance(page, dict) else None,
                "end_cursor": page.get("endCursor") if isinstance(page, dict) else None,
            })
            depth += 1
        for key, value in data.items():
            out.extend(walk_connections(value, key if not path else path + "." + key, depth))
    elif isinstance(data, list):
        for index, item in enumerate(data):
            out.extend(walk_connections(item, "%s[%d]" % (path, index), depth))
    return out


def missing(entry):
    """Items this connection holds and did not return, or None. Pure."""
    total = entry.get("total_count")
    if not isinstance(total, int):
        return None
    return max(0, total - entry.get("returned", 0))


def truncated(entry):
    """Whether this connection stopped short of what it holds. Pure."""
    if entry.get("has_next_page") is True:
        return True
    gap = missing(entry)
    return bool(gap)


def auditable(entry):
    """Whether the response says anything at all about completeness. Pure."""
    return entry.get("total_count") is not None or entry.get("has_next_page") is not None


def resumable(entry):
    """Whether this connection can be continued without refetching its parent."""
    return bool(entry.get("end_cursor")) or entry.get("has_next_page") is not None


def followup_queries(entries):
    """Queries a correct inner walk would cost, from what this response shows.

    One per truncated parent at minimum, and more where the gap is wider than a
    single page of the size that was requested. Pure.
    """
    total = 0
    for entry in entries or []:
        if entry.get("depth", 0) < 1 or not truncated(entry):
            continue
        gap = missing(entry)
        page = entry.get("returned") or 0
        if gap and page > 0:
            total += -(-gap // page)
        else:
            total += 1
    return total


def classify(entries):
    """Classify one response. Pure. Returns (state, detail)."""
    if not entries:
        return ("no-connection-in-the-response",
                "nothing in this response has nodes or edges, so there is no "
                "connection here to be truncated.")
    inner = [e for e in entries if e["depth"] >= 1]
    inner_cut = [e for e in inner if truncated(e)]
    if inner_cut:
        gaps = [missing(e) for e in inner_cut if missing(e) is not None]
        return ("inner-connection-truncated",
                "%d of %d inner connection(s) returned fewer items than they "
                "contain and %s item(s) are missing with no error raised."
                % (len(inner_cut), len(inner),
                   sum(gaps) if gaps else "an unknown number of"))
    blind = [e for e in inner if not auditable(e)]
    if blind:
        return ("inner-connection-unauditable",
                "%d of %d inner connection(s) asked for neither totalCount nor "
                "pageInfo, so this response cannot say whether they truncated."
                % (len(blind), len(inner)))
    outer_cut = [e for e in entries if e["depth"] == 0 and truncated(e)]
    if outer_cut:
        return ("outer-connection-truncated",
                "the outer connection has more pages and every inner connection "
                "in it is complete. This is the truncation people do notice.")
    return ("complete",
            "every connection in this response returned everything it holds, so "
            "a total computed over it really is a total.")


def repair(state):
    """The sentence a reader has to act on. Pure."""
    if state == "inner-connection-truncated":
        return ("add pageInfo { hasNextPage endCursor } to every nested "
                "connection and walk each truncated parent separately with "
                "after: endCursor. An outer loop cannot do this for you.")
    if state == "inner-connection-unauditable":
        return ("add totalCount and pageInfo { hasNextPage endCursor } to the "
                "nested connections first. They cost nothing and without them "
                "nobody can tell whether this response is complete.")
    if state == "outer-connection-truncated":
        return ("follow the outer cursor as you already do, and keep checking "
                "the inner connections on every page: they restart from the "
                "beginning each time the outer one advances.")
    if state == "complete":
        return ("nothing here. Re-run it against a parent that really has more "
                "than one page of children, since a connection that fits cannot "
                "demonstrate a connection that does not.")
    return "point the query at something with a connection in it."


def point_cost(queries):
    """Points this run will spend. Pure."""
    return int(queries or 0) * POINTS_PER_QUERY


def run_query(session, document, variables):
    """Send one query. Returns (status, body-or-None).

    A GraphQL query is a read; POST is only how the document reaches the
    endpoint, which is why the verb is written here beside the URL rather than
    tucked into a constant where it could be mistaken for a write path.
    """
    r = session.post(API + "/graphql",
                     json={"query": document, "variables": variables or {}},
                     timeout=30)
    if r.status_code == 401:
        raise SystemExit("401 from GitHub: GITHUB_TOKEN is missing, malformed "
                         "or revoked")
    try:
        return r.status_code, r.json()
    except ValueError:
        return r.status_code, None


def main():
    ap = argparse.ArgumentParser(description=__doc__)
    ap.add_argument("--login", help="user or organisation to probe with the "
                                    "default query")
    ap.add_argument("--outer", type=int, default=5, help="outer page size")
    ap.add_argument("--inner", type=int, default=5, help="inner page size")
    ap.add_argument("--file", help="a .graphql file to send instead")
    ap.add_argument("--query", help="the document as a string")
    ap.add_argument("--variables", default="{}", help="JSON object of variables")
    args = ap.parse_args()

    if args.file:
        document = Path(args.file).read_text(encoding="utf-8")
    else:
        document = args.query or DEFAULT_QUERY

    try:
        variables = json.loads(args.variables)
    except ValueError:
        log.error("--variables takes a JSON object")
        return 2
    if not isinstance(variables, dict):
        log.error("--variables takes a JSON object")
        return 2
    if not args.file and not args.query:
        if not args.login:
            log.error("--login takes a user or organisation name")
            return 2
        variables.update({"login": args.login, "outer": args.outer,
                          "inner": args.inner})

    why_not = refusal(document)
    if why_not:
        log.error("refusing to send: %s", why_not)
        return 2

    token = os.environ.get("GITHUB_TOKEN")
    if not token:
        log.error("set GITHUB_TOKEN (a read-only token is enough)")
        return 2

    log.info("point cost: %d point(s) against the 5,000/hour GraphQL budget",
             point_cost(1))

    fields = connection_fields(document)
    for f in unauditable(fields):
        log.info("document: %s asks for neither totalCount nor pageInfo, so no "
                 "response can say whether it truncated", f["field"])
    for f in unresumable(fields):
        log.info("document: %s asks for totalCount but not pageInfo, so "
                 "truncation is visible and cannot be resumed without "
                 "refetching the parent", f["field"])

    session = requests.Session()
    session.headers.update({
        "Authorization": "Bearer " + token,
        "Accept": "application/vnd.github+json",
        # GitHub rejects requests with no User-Agent outright.
        "User-Agent": UA,
    })
    status, body = run_query(session, document, variables)
    if not isinstance(body, dict):
        log.error("HTTP %s and no JSON body to read", status)
        return 2
    if body.get("errors"):
        log.error("the query itself failed: %s",
                  json.dumps(body["errors"])[:400])
        return 2

    entries = walk_connections(body.get("data") or {})
    for e in entries:
        gap = missing(e)
        note = "complete"
        if truncated(e):
            note = ("%s missing" % gap) if gap is not None else "more pages"
            if not resumable(e):
                note += ", no cursor"
        log.info("  %-46s depth %d  %d of %s  %s",
                 e["path"], e["depth"], e["returned"],
                 "?" if e["total_count"] is None else e["total_count"], note)

    state, detail = classify(entries)
    log.info("%s: %s", state, detail)
    follow = followup_queries(entries)
    if follow:
        log.info("following them properly costs %d more quer%s, at least one "
                 "per truncated parent", follow, "y" if follow == 1 else "ies")
    log.info("repair: %s", repair(state))

    print(json.dumps({"points_spent": point_cost(1), "state": state,
                      "followup_queries": follow, "connections": entries,
                      "document": fields}, indent=2, default=str))
    return 1 if state in ("inner-connection-truncated",
                          "inner-connection-unauditable") else 0


if __name__ == "__main__":
    sys.exit(main())

/**
 * Find nested GraphQL connections that truncated once per parent.
 *
 * Read only, and queries only. GitHub's GraphQL endpoint takes a document in
 * the request body, so a read is carried by POST there just as a write would
 * be; that is transport, not intent. Any document containing a mutation or a
 * subscription is refused before a socket opens.
 *
 * Environment:
 *   GITHUB_TOKEN      a token with read access to the GraphQL API
 *   GITHUB_LOGIN      user or organisation to probe with the default query
 *   GITHUB_OUTER      outer page size (default 5)
 *   GITHUB_INNER      inner page size (default 5)
 *   GITHUB_QUERY      the document as a string
 *   GITHUB_VARIABLES  JSON object of variables
 */
const API = 'https://api.github.com';
const UA = 'github-graphql-nested/1.0';

export const POINTS_PER_QUERY = 1;

const DEFAULT_QUERY = 'query($login: String!, $outer: Int = 5, $inner: Int = 5) {'
  + ' repositoryOwner(login: $login) {'
  + ' repositories(first: $outer, orderBy: {field: PUSHED_AT, direction: DESC}) {'
  + ' totalCount pageInfo { hasNextPage endCursor }'
  + ' nodes { name issues(first: $inner, states: OPEN) {'
  + ' totalCount nodes { number } } }'
  + ' } } }';

/** Remove GraphQL comments and string literals from a document. Pure. */
export function stripNoise(document) {
  const src = String(document ?? '');
  const out = [];
  let i = 0;
  while (i < src.length) {
    const ch = src[i];
    if (ch === '#') {
      while (i < src.length && src[i] !== '\n') i += 1;
      continue;
    }
    if (src.startsWith('"""', i)) {
      const j = src.indexOf('"""', i + 3);
      i = j < 0 ? src.length : j + 3;
      out.push(' ');
      continue;
    }
    if (ch === '"') {
      i += 1;
      while (i < src.length && src[i] !== '"') i += src[i] === '\\' ? 2 : 1;
      i += 1;
      out.push(' ');
      continue;
    }
    out.push(ch);
    i += 1;
  }
  return out.join('');
}

/** The top-level operations in a document, in order. Pure. */
export function operations(document) {
  const src = `${stripNoise(document)} `;
  const ops = [];
  let depth = 0;
  let word = '';
  let declared = null;
  for (const ch of src) {
    if (/[A-Za-z0-9_]/.test(ch)) { word += ch; continue; }
    if (word) {
      if (depth === 0 && ['query', 'mutation', 'subscription', 'fragment'].includes(word)) {
        declared = word;
      }
      word = '';
    }
    if (ch === '{') {
      if (depth === 0) { ops.push(declared || 'query'); declared = null; }
      depth += 1;
    } else if (ch === '}') {
      depth = Math.max(0, depth - 1);
    }
  }
  return ops;
}

/** Why this document will not be sent, or null if it is a read. Pure. */
export function refusal(document) {
  const ops = operations(document);
  if (ops.length === 0) return 'the document contains no operation to send.';
  for (const kind of ['mutation', 'subscription']) {
    if (ops.includes(kind)) {
      return `the document contains a ${kind}. This script sends queries only: `
        + 'a query is a read, and the section it belongs to promises its '
        + 'scripts never write.';
    }
  }
  return null;
}

/** A selection set with everything nested inside it blanked out. Pure. */
export function outerText(block) {
  const out = [];
  let depth = 0;
  for (const ch of String(block ?? '')) {
    if (ch === '{') { depth += 1; out.push(' '); continue; }
    if (ch === '}') { depth = Math.max(0, depth - 1); out.push(' '); continue; }
    out.push(depth === 0 ? ch : ' ');
  }
  return out.join('');
}

/** Every connection in the document, found by shape. Pure. */
export function connectionFields(document, depthIn = 0, stripped = false) {
  const src = stripped ? String(document ?? '') : stripNoise(document);
  const out = [];
  let i = 0;
  let word = '';
  let field = '';
  while (i < src.length) {
    const ch = src[i];
    if (/[A-Za-z0-9_]/.test(ch)) { word += ch; i += 1; continue; }
    // The field name has to survive the whitespace and the argument list
    // between it and its selection set, so it is remembered rather than read
    // off whatever happens to precede the brace.
    if (word) { field = word; word = ''; }
    if (ch === '(') {
      let j = i;
      let level = 0;
      while (j < src.length) {
        if (src[j] === '(') level += 1;
        else if (src[j] === ')') { level -= 1; if (level === 0) break; }
        j += 1;
      }
      i = j + 1;
      continue;
    }
    if (ch === '{') {
      let j = i;
      let level = 0;
      while (j < src.length) {
        if (src[j] === '{') level += 1;
        else if (src[j] === '}') { level -= 1; if (level === 0) break; }
        j += 1;
      }
      const block = src.slice(i + 1, j);
      const own = outerText(block).split(/\s+/).filter(Boolean);
      if (field && (own.includes('nodes') || own.includes('edges'))) {
        out.push({
          field,
          depth: depthIn,
          has_page_info: own.includes('pageInfo'),
          has_total_count: own.includes('totalCount'),
        });
        out.push(...connectionFields(block, depthIn + 1, true));
      } else {
        out.push(...connectionFields(block, depthIn, true));
      }
      i = j + 1;
      field = '';
      continue;
    }
    if (ch === '}') field = '';
    i += 1;
  }
  return out;
}

/** Inner connections that asked for neither totalCount nor pageInfo. Pure. */
export function unauditable(fields) {
  return (fields || []).filter((f) => f.depth >= 1 && !f.has_total_count && !f.has_page_info);
}

/** Inner connections that can be seen to truncate but not continued. Pure. */
export function unresumable(fields) {
  return (fields || []).filter((f) => f.depth >= 1 && f.has_total_count && !f.has_page_info);
}

/** Whether a decoded object is a connection. Pure. */
export function isConnection(value) {
  if (!value || typeof value !== 'object' || Array.isArray(value)) return false;
  return Array.isArray(value.nodes) || Array.isArray(value.edges);
}

/** Every connection in a decoded response, with its path and depth. Pure. */
export function walkConnections(data, path = '', depth = 0) {
  const out = [];
  if (Array.isArray(data)) {
    data.forEach((item, index) => {
      out.push(...walkConnections(item, `${path}[${index}]`, depth));
    });
    return out;
  }
  if (!data || typeof data !== 'object') return out;
  let next = depth;
  if (isConnection(data)) {
    const items = Array.isArray(data.nodes) ? data.nodes : (data.edges || []);
    const page = (data.pageInfo && typeof data.pageInfo === 'object') ? data.pageInfo : null;
    out.push({
      path: path || '(root)',
      depth,
      returned: items.length,
      total_count: Number.isInteger(data.totalCount) ? data.totalCount : null,
      has_next_page: page ? page.hasNextPage : null,
      end_cursor: page ? page.endCursor : null,
    });
    next = depth + 1;
  }
  for (const [key, value] of Object.entries(data)) {
    out.push(...walkConnections(value, path ? `${path}.${key}` : key, next));
  }
  return out;
}

/** Items this connection holds and did not return, or null. Pure. */
export function missing(entry) {
  const total = entry && entry.total_count;
  if (!Number.isInteger(total)) return null;
  return Math.max(0, total - ((entry && entry.returned) || 0));
}

/** Whether this connection stopped short of what it holds. Pure. */
export function truncated(entry) {
  if (entry && entry.has_next_page === true) return true;
  return !!missing(entry);
}

/** Whether the response says anything at all about completeness. Pure. */
export function auditable(entry) {
  if (!entry) return false;
  return entry.total_count !== null || (entry.has_next_page !== null
    && entry.has_next_page !== undefined);
}

/** Whether this connection can be continued without refetching its parent. */
export function resumable(entry) {
  if (!entry) return false;
  return !!entry.end_cursor || (entry.has_next_page !== null
    && entry.has_next_page !== undefined);
}

/** Queries a correct inner walk would cost, from what this response shows. */
export function followupQueries(entries) {
  let total = 0;
  for (const entry of entries || []) {
    if ((entry.depth || 0) < 1 || !truncated(entry)) continue;
    const gap = missing(entry);
    const page = entry.returned || 0;
    if (gap && page > 0) total += Math.ceil(gap / page);
    else total += 1;
  }
  return total;
}

/** Classify one response. Pure. Returns [state, detail]. */
export function classify(entries) {
  if (!entries || entries.length === 0) {
    return ['no-connection-in-the-response', 'nothing in this response has '
      + 'nodes or edges, so there is no connection here to be truncated.'];
  }
  const inner = entries.filter((e) => e.depth >= 1);
  const innerCut = inner.filter((e) => truncated(e));
  if (innerCut.length) {
    const gaps = innerCut.map((e) => missing(e)).filter((g) => g !== null);
    const sum = gaps.length ? gaps.reduce((a, b) => a + b, 0) : 'an unknown number of';
    return ['inner-connection-truncated',
      `${innerCut.length} of ${inner.length} inner connection(s) returned fewer `
      + `items than they contain and ${sum} item(s) are missing with no error raised.`];
  }
  const blind = inner.filter((e) => !auditable(e));
  if (blind.length) {
    return ['inner-connection-unauditable',
      `${blind.length} of ${inner.length} inner connection(s) asked for neither `
      + 'totalCount nor pageInfo, so this response cannot say whether they truncated.'];
  }
  const outerCut = entries.filter((e) => e.depth === 0 && truncated(e));
  if (outerCut.length) {
    return ['outer-connection-truncated', 'the outer connection has more pages '
      + 'and every inner connection in it is complete. This is the truncation '
      + 'people do notice.'];
  }
  return ['complete', 'every connection in this response returned everything it '
    + 'holds, so a total computed over it really is a total.'];
}

/** The sentence a reader has to act on. Pure. */
export function repair(state) {
  if (state === 'inner-connection-truncated') {
    return 'add pageInfo { hasNextPage endCursor } to every nested connection '
      + 'and walk each truncated parent separately with after: endCursor. An '
      + 'outer loop cannot do this for you.';
  }
  if (state === 'inner-connection-unauditable') {
    return 'add totalCount and pageInfo { hasNextPage endCursor } to the nested '
      + 'connections first. They cost nothing and without them nobody can tell '
      + 'whether this response is complete.';
  }
  if (state === 'outer-connection-truncated') {
    return 'follow the outer cursor as you already do, and keep checking the '
      + 'inner connections on every page: they restart from the beginning each '
      + 'time the outer one advances.';
  }
  if (state === 'complete') {
    return 'nothing here. Re-run it against a parent that really has more than '
      + 'one page of children, since a connection that fits cannot demonstrate '
      + 'a connection that does not.';
  }
  return 'point the query at something with a connection in it.';
}

/** Points this run will spend. Pure. */
export function pointCost(queries) {
  return Number(queries || 0) * POINTS_PER_QUERY;
}

function headers(token) {
  return {
    Authorization: `Bearer ${token}`,
    Accept: 'application/vnd.github+json',
    'Content-Type': 'application/json',
    'User-Agent': UA,
  };
}

async function runQuery(token, document, variables) {
  const res = await fetch(`${API}/graphql`, {
    // A GraphQL query is a read. POST is only how the document reaches the
    // endpoint, and refusal() has already rejected anything that is not a read.
    method: 'POST',
    headers: headers(token),
    body: JSON.stringify({ query: document, variables: variables || {} }),
  });
  let body = null;
  try { body = await res.json(); } catch { body = null; }
  return { status: res.status, body };
}

async function main() {
  const token = process.env.GITHUB_TOKEN;
  if (!token) {
    console.error('set GITHUB_TOKEN (a read-only token is enough)');
    process.exitCode = 2;
    return;
  }
  const document = process.env.GITHUB_QUERY || DEFAULT_QUERY;
  let variables = {};
  try { variables = JSON.parse(process.env.GITHUB_VARIABLES || '{}'); } catch {
    console.error('GITHUB_VARIABLES takes a JSON object');
    process.exitCode = 2;
    return;
  }
  if (!process.env.GITHUB_QUERY) {
    if (!process.env.GITHUB_LOGIN) {
      console.error('set GITHUB_LOGIN to a user or organisation name');
      process.exitCode = 2;
      return;
    }
    variables.login = process.env.GITHUB_LOGIN;
    variables.outer = Number(process.env.GITHUB_OUTER || 5);
    variables.inner = Number(process.env.GITHUB_INNER || 5);
  }

  const whyNot = refusal(document);
  if (whyNot) {
    console.error(`refusing to send: ${whyNot}`);
    process.exitCode = 2;
    return;
  }

  console.log(`point cost: ${pointCost(1)} point(s) against the 5,000/hour GraphQL budget`);
  const fields = connectionFields(document);
  for (const f of unauditable(fields)) {
    console.log(`document: ${f.field} asks for neither totalCount nor pageInfo, `
      + 'so no response can say whether it truncated');
  }
  for (const f of unresumable(fields)) {
    console.log(`document: ${f.field} asks for totalCount but not pageInfo, so `
      + 'truncation is visible and cannot be resumed without refetching the parent');
  }

  const { status, body } = await runQuery(token, document, variables);
  if (!body || typeof body !== 'object') {
    console.error(`HTTP ${status} and no JSON body to read`);
    process.exitCode = 2;
    return;
  }
  if (Array.isArray(body.errors) && body.errors.length) {
    console.error(`the query itself failed: ${JSON.stringify(body.errors).slice(0, 400)}`);
    process.exitCode = 2;
    return;
  }

  const entries = walkConnections(body.data || {});
  for (const e of entries) {
    const gap = missing(e);
    let note = 'complete';
    if (truncated(e)) {
      note = gap !== null ? `${gap} missing` : 'more pages';
      if (!resumable(e)) note += ', no cursor';
    }
    console.log(`  ${e.path}  depth ${e.depth}  ${e.returned} of `
      + `${e.total_count === null ? '?' : e.total_count}  ${note}`);
  }

  const [state, detail] = classify(entries);
  console.log(`${state}: ${detail}`);
  const follow = followupQueries(entries);
  if (follow) {
    console.log(`following them properly costs ${follow} more `
      + `${follow === 1 ? 'query' : 'queries'}, at least one per truncated parent`);
  }
  console.log(`repair: ${repair(state)}`);

  console.log(JSON.stringify({
    points_spent: pointCost(1), state, followup_queries: follow,
    connections: entries, document: fields,
  }, null, 2));
  process.exitCode = ['inner-connection-truncated',
    'inner-connection-unauditable'].includes(state) ? 1 : 0;
}

// Guarded so importing this file from the test runner does not start a run.
if (import.meta.url === `file://${process.argv[1]}`) {
  main().catch((err) => { console.error(err.message); process.exitCode = 2; });
}

from github_graphql_nested import (
    POINTS_PER_QUERY, auditable, classify, connection_fields, followup_queries,
    is_connection, missing, operations, outer_text, point_cost, refusal, repair,
    resumable, truncated, unauditable, unresumable, walk_connections,
)

DATA = {"repositoryOwner": {"repositories": {
    "totalCount": 218,
    "pageInfo": {"hasNextPage": True, "endCursor": "Y3Vyc29yOjU="},
    "nodes": [
        {"name": "monorepo",
         "issues": {"totalCount": 406, "nodes": [{"number": 1}, {"number": 2}]}},
        {"name": "tiny",
         "issues": {"totalCount": 2, "nodes": [{"number": 9}, {"number": 10}]}},
    ],
}}}

NESTED_QUERY = ("query { repositoryOwner(login: \"acme\") {"
                " repositories(first: 5) { totalCount"
                " pageInfo { hasNextPage endCursor }"
                " nodes { name issues(first: 5) { totalCount nodes { number } } } } } }")


def test_the_walk_finds_inner_connections_by_path():
    entries = walk_connections(DATA)
    paths = [e["path"] for e in entries]
    assert "repositoryOwner.repositories" in paths
    assert "repositoryOwner.repositories.nodes[0].issues" in paths
    assert "repositoryOwner.repositories.nodes[1].issues" in paths


def test_an_inner_connection_is_deeper_than_the_one_containing_it():
    by_path = {e["path"]: e for e in walk_connections(DATA)}
    assert by_path["repositoryOwner.repositories"]["depth"] == 0
    assert by_path["repositoryOwner.repositories.nodes[0].issues"]["depth"] == 1


def test_truncation_is_measured_per_parent_and_not_in_total():
    by_path = {e["path"]: e for e in walk_connections(DATA)}
    big = by_path["repositoryOwner.repositories.nodes[0].issues"]
    small = by_path["repositoryOwner.repositories.nodes[1].issues"]
    assert missing(big) == 404
    assert truncated(big)
    assert missing(small) == 0
    assert not truncated(small)


def test_has_next_page_alone_is_enough_to_call_it_truncated():
    entry = {"depth": 1, "returned": 100, "total_count": None,
             "has_next_page": True, "end_cursor": "abc"}
    assert truncated(entry)
    assert missing(entry) is None
    assert auditable(entry)


def test_a_connection_with_neither_field_cannot_be_judged_at_all():
    entry = {"depth": 1, "returned": 100, "total_count": None,
             "has_next_page": None, "end_cursor": None}
    assert not auditable(entry)
    assert not truncated(entry)
    state, detail = classify([{"depth": 0, "returned": 5, "total_count": 5,
                               "has_next_page": False, "end_cursor": None}, entry])
    assert state == "inner-connection-unauditable"
    assert "neither totalCount nor pageInfo" in detail


def test_seeing_the_gap_and_being_able_to_resume_it_are_different():
    seen_only = {"depth": 1, "returned": 5, "total_count": 406,
                 "has_next_page": None, "end_cursor": None}
    assert truncated(seen_only)
    assert not resumable(seen_only)
    assert resumable({"depth": 1, "returned": 5, "total_count": 406,
                      "has_next_page": True, "end_cursor": "abc"})


def test_the_inner_truncation_outranks_the_outer_one():
    state, detail = classify(walk_connections(DATA))
    assert state == "inner-connection-truncated"
    assert "404" in detail
    assert "after: endCursor" in repair(state)


def test_an_outer_only_truncation_is_named_as_the_one_people_notice():
    data = {"repositories": {"totalCount": 218,
                             "pageInfo": {"hasNextPage": True, "endCursor": "c"},
                             "nodes": [{"name": "tiny",
                                        "issues": {"totalCount": 2,
                                                   "nodes": [{"number": 1}, {"number": 2}]}}]}}
    state, detail = classify(walk_connections(data))
    assert state == "outer-connection-truncated"
    assert "do notice" in detail


def test_a_complete_response_is_not_reported_as_a_finding():
    data = {"repositories": {"totalCount": 1,
                             "pageInfo": {"hasNextPage": False, "endCursor": None},
                             "nodes": [{"name": "tiny",
                                        "issues": {"totalCount": 1,
                                                   "nodes": [{"number": 1}]}}]}}
    assert classify(walk_connections(data))[0] == "complete"
    assert classify([])[0] == "no-connection-in-the-response"


def test_an_inner_page_info_is_never_credited_to_its_parent():
    doc = ("query { a(first: 10) { totalCount nodes {"
           " b(first: 10) { pageInfo { hasNextPage } nodes { id } } } } }")
    fields = {f["field"]: f for f in connection_fields(doc)}
    assert fields["a"]["has_total_count"] and not fields["a"]["has_page_info"]
    assert fields["b"]["has_page_info"] and not fields["b"]["has_total_count"]
    assert fields["a"]["depth"] == 0 and fields["b"]["depth"] == 1
    assert "pageInfo" not in outer_text(" totalCount nodes { pageInfo { x } } ")


def test_the_document_audit_names_what_cannot_be_checked_or_resumed():
    fields = connection_fields(NESTED_QUERY)
    assert [f["field"] for f in unresumable(fields)] == ["issues"]
    assert unauditable(fields) == []
    bare = "query { a(first: 5) { nodes { b(first: 5) { nodes { id } } } } }"
    assert [f["field"] for f in unauditable(connection_fields(bare))] == ["b"]


def test_a_connection_is_recognised_by_nodes_or_edges():
    assert is_connection({"nodes": []})
    assert is_connection({"edges": []})
    assert not is_connection({"nodes": 3})
    assert not is_connection({"name": "monorepo"})


def test_the_cost_of_doing_it_properly_is_counted_before_the_loop_is_written():
    assert followup_queries(walk_connections(DATA)) == 202
    assert followup_queries([]) == 0
    assert POINTS_PER_QUERY == 1
    assert point_cost(1) == 1
    assert point_cost(0) == 0


def test_the_script_refuses_to_send_a_mutation():
    assert operations("query Q { viewer { login } }") == ["query"]
    assert refusal("mutation M { addStar(input: {}) { clientMutationId } }")
    assert refusal("subscription S { thing { id } }")
    assert refusal(NESTED_QUERY) is None

import { test } from 'node:test';
import assert from 'node:assert/strict';
import {
  POINTS_PER_QUERY, auditable, classify, connectionFields, followupQueries,
  isConnection, missing, operations, outerText, pointCost, refusal, repair,
  resumable, truncated, unauditable, unresumable, walkConnections,
} from './github-graphql-nested.mjs';

const DATA = {
  repositoryOwner: {
    repositories: {
      totalCount: 218,
      pageInfo: { hasNextPage: true, endCursor: 'Y3Vyc29yOjU=' },
      nodes: [
        { name: 'monorepo', issues: { totalCount: 406, nodes: [{ number: 1 }, { number: 2 }] } },
        { name: 'tiny', issues: { totalCount: 2, nodes: [{ number: 9 }, { number: 10 }] } },
      ],
    },
  },
};

const NESTED_QUERY = 'query { repositoryOwner(login: "acme") {'
  + ' repositories(first: 5) { totalCount'
  + ' pageInfo { hasNextPage endCursor }'
  + ' nodes { name issues(first: 5) { totalCount nodes { number } } } } } }';

test('the walk finds inner connections by path', () => {
  const paths = walkConnections(DATA).map((e) => e.path);
  assert.ok(paths.includes('repositoryOwner.repositories'));
  assert.ok(paths.includes('repositoryOwner.repositories.nodes[0].issues'));
  assert.ok(paths.includes('repositoryOwner.repositories.nodes[1].issues'));
});

test('an inner connection is deeper than the one containing it', () => {
  const byPath = Object.fromEntries(walkConnections(DATA).map((e) => [e.path, e]));
  assert.equal(byPath['repositoryOwner.repositories'].depth, 0);
  assert.equal(byPath['repositoryOwner.repositories.nodes[0].issues'].depth, 1);
});

test('truncation is measured per parent and not in total', () => {
  const byPath = Object.fromEntries(walkConnections(DATA).map((e) => [e.path, e]));
  const big = byPath['repositoryOwner.repositories.nodes[0].issues'];
  const small = byPath['repositoryOwner.repositories.nodes[1].issues'];
  assert.equal(missing(big), 404);
  assert.ok(truncated(big));
  assert.equal(missing(small), 0);
  assert.ok(!truncated(small));
});

test('has next page alone is enough to call it truncated', () => {
  const entry = {
    depth: 1, returned: 100, total_count: null, has_next_page: true, end_cursor: 'abc',
  };
  assert.ok(truncated(entry));
  assert.equal(missing(entry), null);
  assert.ok(auditable(entry));
});

test('a connection with neither field cannot be judged at all', () => {
  const entry = {
    depth: 1, returned: 100, total_count: null, has_next_page: null, end_cursor: null,
  };
  assert.ok(!auditable(entry));
  assert.ok(!truncated(entry));
  const [state, detail] = classify([{
    depth: 0, returned: 5, total_count: 5, has_next_page: false, end_cursor: null,
  }, entry]);
  assert.equal(state, 'inner-connection-unauditable');
  assert.match(detail, /neither totalCount nor pageInfo/);
});

test('seeing the gap and being able to resume it are different', () => {
  const seenOnly = {
    depth: 1, returned: 5, total_count: 406, has_next_page: null, end_cursor: null,
  };
  assert.ok(truncated(seenOnly));
  assert.ok(!resumable(seenOnly));
  assert.ok(resumable({
    depth: 1, returned: 5, total_count: 406, has_next_page: true, end_cursor: 'abc',
  }));
});

test('the inner truncation outranks the outer one', () => {
  const [state, detail] = classify(walkConnections(DATA));
  assert.equal(state, 'inner-connection-truncated');
  assert.match(detail, /404/);
  assert.match(repair(state), /after: endCursor/);
});

test('an outer only truncation is named as the one people notice', () => {
  const data = {
    repositories: {
      totalCount: 218,
      pageInfo: { hasNextPage: true, endCursor: 'c' },
      nodes: [{ name: 'tiny', issues: { totalCount: 2, nodes: [{ number: 1 }, { number: 2 }] } }],
    },
  };
  const [state, detail] = classify(walkConnections(data));
  assert.equal(state, 'outer-connection-truncated');
  assert.match(detail, /do notice/);
});

test('a complete response is not reported as a finding', () => {
  const data = {
    repositories: {
      totalCount: 1,
      pageInfo: { hasNextPage: false, endCursor: null },
      nodes: [{ name: 'tiny', issues: { totalCount: 1, nodes: [{ number: 1 }] } }],
    },
  };
  assert.equal(classify(walkConnections(data))[0], 'complete');
  assert.equal(classify([])[0], 'no-connection-in-the-response');
});

test('an inner page info is never credited to its parent', () => {
  const doc = 'query { a(first: 10) { totalCount nodes {'
    + ' b(first: 10) { pageInfo { hasNextPage } nodes { id } } } } }';
  const fields = Object.fromEntries(connectionFields(doc).map((f) => [f.field, f]));
  assert.ok(fields.a.has_total_count && !fields.a.has_page_info);
  assert.ok(fields.b.has_page_info && !fields.b.has_total_count);
  assert.equal(fields.a.depth, 0);
  assert.equal(fields.b.depth, 1);
  assert.ok(!outerText(' totalCount nodes { pageInfo { x } } ').includes('pageInfo'));
});

test('the document audit names what cannot be checked or resumed', () => {
  const fields = connectionFields(NESTED_QUERY);
  assert.deepEqual(unresumable(fields).map((f) => f.field), ['issues']);
  assert.deepEqual(unauditable(fields), []);
  const bare = 'query { a(first: 5) { nodes { b(first: 5) { nodes { id } } } } }';
  assert.deepEqual(unauditable(connectionFields(bare)).map((f) => f.field), ['b']);
});

test('a connection is recognised by nodes or edges', () => {
  assert.ok(isConnection({ nodes: [] }));
  assert.ok(isConnection({ edges: [] }));
  assert.ok(!isConnection({ nodes: 3 }));
  assert.ok(!isConnection({ name: 'monorepo' }));
});

test('the cost of doing it properly is counted before the loop is written', () => {
  assert.equal(followupQueries(walkConnections(DATA)), 202);
  assert.equal(followupQueries([]), 0);
  assert.equal(POINTS_PER_QUERY, 1);
  assert.equal(pointCost(1), 1);
  assert.equal(pointCost(0), 0);
});

test('the script refuses to send a mutation', () => {
  assert.deepEqual(operations('query Q { viewer { login } }'), ['query']);
  assert.ok(refusal('mutation M { addStar(input: {}) { clientMutationId } }'));
  assert.ok(refusal('subscription S { thing { id } }'));
  assert.equal(refusal(NESTED_QUERY), null);
});