SocialPub/tools/pasture/town/backlog.py

"""docs/INTEROP-BACKLOG.md from the latest town results and the bash scenarios: new failures (to fix or to record in
gaps.json), known gaps by phase, known gaps now passing (close them), and the scenarios' failures.

  town.sh backlog [run] [--write]"""
import json
import os
import sys
import time
from collections import defaultdict

import check
import seed

DOCS = os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))), "docs")


def load_jsonl(path):
	if not os.path.exists(path):
		return []
	with open(path) as f:
		return [json.loads(line) for line in f if line.strip()]


def main(args):
	name = next((a for a in args if not a.startswith("--")), "village")
	run_dir = seed.run_dir(name)
	with open(os.path.join(run_dir, "results.json")) as f:
		result = json.load(f)
	gaps = {g["id"]: g for g in check.load_gaps()}
	scenarios = load_jsonl(os.path.join(seed.OUT, "scenarios.jsonl"))
	checks = result["checks"]

	new = defaultdict(list)
	known = defaultdict(list)
	closing = defaultdict(list)
	for c in checks:
		if c["status"] == "fail":
			new[c["cell"]].append(c)
		elif c["status"] == "xfail":
			known[c["gap"]].append(c)
		elif c["status"] == "xpass":
			closing[c["gap"]].append(c)

	lines = ["# Interop backlog", "",
			 f"Generated by `tools/pasture/town.sh backlog --write` from `{os.path.basename(run_dir)}` on "
			 f"{time.strftime('%Y-%m-%d', time.gmtime())}. Do not edit by hand: fix the cause, or record a known gap in "
			 "`tools/pasture/town/gaps.json`, and generate it again.", "",
			 f"Checks: {sum(1 for c in checks if c['status'] == 'pass')} pass, {sum(1 for c in checks if c['status'] == 'fail')} "
			 f"fail, {sum(1 for c in checks if c['status'] == 'xfail')} known gaps, "
			 f"{sum(1 for c in checks if c['status'] == 'xpass')} known gaps now passing.", ""]
	lines += ["## New failures", ""]
	if not new:
		lines.append("None.")
	for cell, cs in sorted(new.items(), key=lambda kv: -len(kv[1])):
		ex = cs[0]
		got = " ".join(str(ex.get("got")).replace("`", "'").split())[:140]
		lines.append(f"- **`{cell}`**: {len(cs)} failing. Example: {ex.get('ref') or ''} expected `{ex.get('expect')}`, "
					 f"got `{got}`" + (f" ({ex['how']})" if ex.get("how") else "") + ".")
	lines += ["", "## Known gaps still failing", ""]
	if not known:
		lines.append("None.")
	for gid, cs in sorted(known.items()):
		g = gaps.get(gid, {})
		lines.append(f"- **{gid}** ({g.get('kind')}, {g.get('phase')}): {g.get('title')}. {len(cs)} checks. "
					 + (f"Code: `{g['code']}`." if g.get("code") else ""))
	lines += ["", "## Known gaps now passing (close them)", ""]
	if not closing:
		lines.append("None.")
	for gid, cs in sorted(closing.items()):
		lines.append(f"- **{gid}**: {gaps.get(gid, {}).get('title')}. {len(cs)} checks pass.")
	lines += ["", "## Scenario failures", ""]
	bad = [s for s in scenarios if s["status"] == "fail"]
	lines += [f"- {s['peer']}: {s['check']}" for s in bad] or ["None."]
	text = "\n".join(lines) + "\n"
	if "--write" in args:
		with open(os.path.join(DOCS, "INTEROP-BACKLOG.md"), "w") as f:
			f.write(text)
		print(os.path.join(DOCS, "INTEROP-BACKLOG.md"))
	else:
		print(text)
	return 0


if __name__ == "__main__":
	sys.exit(main(sys.argv[1:]))