#!/usr/bin/env python3
"""Compile session evidence and repo baseline into a procedural-memory report."""

from __future__ import annotations

import argparse
import dataclasses
import datetime as dt
import json
import os
import pathlib
import re
import sys

from export_gates import parse_gates, render_registry
from export_skill_context import write_context
from scrub_secrets import scrub
from validate_outputs import validate


DOMAIN_RULES_DIR = pathlib.Path(__file__).resolve().parent.parent / "domain-rules"
DEFAULT_DOMAIN_PRESET = "generic"

STRONG_SIGNAL_PATTERNS = [
    r"\bverify\b",
    r"\bverifiser\b",
    r"\bvalider\b",
    r"ikke spekuler",
    r"are you sure",
    r"\bfaktisk\b",
    r"\bsjekk\b",
    r"double[- ]check",
    r"\breadback\b",
    r"\bsmoke\b",
    r"\bblock",
    r"\bfeil\b",
    r"\bwrong\b",
    r"safe to end",
]

SESSION_REF_RE = re.compile(r"\s+\[session_ref=([^\]]+)\]\s*$")


@dataclasses.dataclass(frozen=True)
class CorpusMessage:
    role: str
    text: str
    session_ref: str = "unknown"


def load_json(path: pathlib.Path) -> dict:
    return json.loads(path.read_text(encoding="utf-8"))


def load_optional_json(path: str | None) -> dict:
    if not path:
        return {}
    candidate = pathlib.Path(path)
    if not candidate.exists():
        return {}
    return load_json(candidate)


def packaged_domain_rules_path(preset: str) -> pathlib.Path:
    safe = re.sub(r"[^A-Za-z0-9._-]+", "-", preset.strip()).strip("-")
    if not safe:
        raise ValueError("domain preset must not be empty")
    path = DOMAIN_RULES_DIR / f"{safe}.json"
    if not path.exists():
        raise ValueError(f"unknown domain preset: {preset}")
    return path


def normalize_domain_rules(payload: object, source: pathlib.Path) -> list[dict]:
    if isinstance(payload, dict):
        rules = payload.get("rules")
    else:
        rules = payload
    if not isinstance(rules, list):
        raise ValueError(f"{source} must contain a JSON object with rules[] or a rules list")

    normalized: list[dict] = []
    seen: set[str] = set()
    required = ("domain", "category", "patterns", "failure_signal", "gate")
    for index, row in enumerate(rules, start=1):
        if not isinstance(row, dict):
            raise ValueError(f"{source} rule {index} must be an object")
        missing = [key for key in required if key not in row]
        if missing:
            raise ValueError(f"{source} rule {index} missing: {', '.join(missing)}")
        domain = str(row["domain"]).strip()
        category = str(row["category"]).strip()
        failure_signal = str(row["failure_signal"]).strip()
        gate = str(row["gate"]).strip()
        patterns = row["patterns"]
        if not domain or not category or not failure_signal or not gate:
            raise ValueError(f"{source} rule {index} has an empty required string")
        if domain in seen:
            raise ValueError(f"{source} has duplicate domain: {domain}")
        if not isinstance(patterns, list) or not patterns:
            raise ValueError(f"{source} rule {domain} patterns must be a non-empty list")
        normalized_patterns = [str(pattern) for pattern in patterns if str(pattern).strip()]
        if len(normalized_patterns) != len(patterns):
            raise ValueError(f"{source} rule {domain} has an empty pattern")
        for pattern in normalized_patterns:
            try:
                re.compile(pattern)
            except re.error as error:
                raise ValueError(f"{source} rule {domain} invalid regex {pattern!r}: {error}") from error
        normalized.append(
            {
                "domain": domain,
                "category": category,
                "patterns": normalized_patterns,
                "failure_signal": failure_signal,
                "gate": gate,
            }
        )
        seen.add(domain)
    return normalized


def load_domain_rules(path: str | pathlib.Path) -> list[dict]:
    candidate = pathlib.Path(path).expanduser()
    return normalize_domain_rules(load_json(candidate), candidate)


def repo_config_domain_rules_path(baseline: dict) -> pathlib.Path | None:
    repo = baseline.get("repo")
    if not repo:
        return None
    config = pathlib.Path(str(repo)).expanduser() / ".agent-learning.json"
    if not config.exists():
        return None
    try:
        payload = load_json(config)
    except (json.JSONDecodeError, OSError):
        return None
    path = payload.get("domain_rules")
    return pathlib.Path(str(path)).expanduser() if path else None


def resolve_domain_rules(
    baseline: dict,
    explicit_path: str | None = None,
    preset: str = DEFAULT_DOMAIN_PRESET,
) -> tuple[list[dict], pathlib.Path]:
    if explicit_path:
        source = pathlib.Path(explicit_path).expanduser()
    elif os.environ.get("AGENT_LEARNING_DOMAIN_RULES"):
        source = pathlib.Path(os.environ["AGENT_LEARNING_DOMAIN_RULES"]).expanduser()
    else:
        source = repo_config_domain_rules_path(baseline) or packaged_domain_rules_path(preset)
    return load_domain_rules(source), source


def quote(text: str) -> str:
    # Scrub before quoting so a caller that bypasses extract_sessions (and its
    # upstream scrub) cannot land secret-shaped text in the report. Anything
    # that scrub touches will get a [REDACTED:*] marker, which quote_is_useful
    # then rejects, so the quote is silently dropped instead of persisted.
    text = scrub(text)
    words = text.strip().replace("\n", " ").split()
    return " ".join(words[:25]).replace('"', "'")


def quote_is_useful(value: str) -> bool:
    return len(value.split()) >= 3 and "[REDACTED:" not in value


def quote_score(value: str, rule: dict) -> int:
    lowered = value.lower()
    score = 0
    if 5 <= len(value.split()) <= 25:
        score += 2
    if any(re.search(pattern, lowered, re.I) for pattern in STRONG_SIGNAL_PATTERNS):
        score += 5
    if any(re.search(pattern, value, re.I) for pattern in rule["patterns"]):
        score += 2
    if value.endswith("?"):
        score += 1
    if len(value.split()) <= 3:
        score -= 2
    return score


def is_agent_command(line: str) -> bool:
    return "<command-message>" in line or "<local-command" in line


def is_context_dump(line: str) -> bool:
    stripped = line.strip()
    if len(stripped) > 500:
        return True
    context_markers = (
        "# AGENTS.md instructions",
        "<environment_context>",
        "</environment_context>",
        "<INSTRUCTIONS>",
        "</INSTRUCTIONS>",
        "@/",
    )
    return any(marker in stripped for marker in context_markers)


def split_session_ref(text: str) -> tuple[str, str]:
    match = SESSION_REF_RE.search(text)
    if not match:
        return text.strip(), "unknown"
    return text[: match.start()].strip(), match.group(1)


def corpus_messages(corpus: str) -> list[CorpusMessage]:
    messages: list[CorpusMessage] = []
    for line in corpus.splitlines():
        if line.startswith("user: "):
            text, ref = split_session_ref(line[6:].strip())
            messages.append(CorpusMessage("user", text, ref))
        elif line.startswith("assistant: "):
            text, ref = split_session_ref(line[11:].strip())
            messages.append(CorpusMessage("assistant", text, ref))
    return messages


def user_lines(corpus: str) -> list[str]:
    return [message.text for message in corpus_messages(corpus) if message.role == "user"]


def assistant_lines(corpus: str) -> list[str]:
    return [message.text for message in corpus_messages(corpus) if message.role == "assistant"]


def meta_lines(corpus: str) -> list[str]:
    return [line[6:].strip() for line in corpus.splitlines() if line.startswith("meta: ")]


def clean_user_messages(corpus: str) -> list[CorpusMessage]:
    return [
        message
        for message in corpus_messages(corpus)
        if message.role == "user" and not is_agent_command(message.text) and not is_context_dump(message.text)
    ]


def clean_user_lines(corpus: str) -> list[str]:
    return [message.text for message in clean_user_messages(corpus)]


def find_preference_evidence(lines: list[str], domain_rules: list[dict] | None = None) -> list[str]:
    if domain_rules is None:
        domain_rules, _source = resolve_domain_rules({})
    evidence: list[str] = []
    for line in lines:
        if is_agent_command(line):
            continue
        if any(re.search(pattern, line, re.I) for rule in domain_rules for pattern in rule["patterns"]):
            evidence.append(line)
    return evidence[:8]


def blank_event(rule: dict) -> dict:
    return {
        "domain": rule["domain"],
        "category": rule["category"],
        "failure_signal": rule["failure_signal"],
        "gate": rule["gate"],
        "quotes": [],
        "count": 0,
        "source_count": 0,
    }


def classify_events(
    baseline: dict,
    messages: list[CorpusMessage] | list[str],
    domain_rules: list[dict] | None = None,
) -> list[dict]:
    if domain_rules is None:
        domain_rules, _source = resolve_domain_rules(baseline)
    events = {rule["domain"]: blank_event(rule) for rule in domain_rules}
    normalized_messages = [
        message if isinstance(message, CorpusMessage) else CorpusMessage("user", str(message), "unknown")
        for message in messages
    ]
    for message in normalized_messages:
        line = message.text
        for rule in domain_rules:
            if any(re.search(pattern, line, re.I) for pattern in rule["patterns"]):
                event = events[rule["domain"]]
                event["count"] += 1
                event.setdefault("session_refs", set()).add(message.session_ref)
                candidate = quote(line)
                if quote_is_useful(candidate):
                    event["quotes"].append((quote_score(candidate, rule), candidate))

    if baseline.get("source_files") or baseline.get("skills"):
        event = events.get("repo-workflow")
        if event:
            event["source_count"] += len(baseline.get("source_files", [])) + len(baseline.get("skills", []))
            event["count"] = max(event["count"], 1)
    if baseline.get("validation_commands"):
        event = events.get("validation")
        if event:
            event["source_count"] += len(baseline.get("validation_commands", []))
            event["count"] = max(event["count"], 1)

    selected = []
    for rule in domain_rules:
        event = events[rule["domain"]]
        if event["count"] <= 0:
            continue
        ranked_quotes = sorted(event["quotes"], key=lambda item: (-item[0], item[1]))
        event["quotes"] = [candidate for _score, candidate in ranked_quotes[:2]]
        event["session_refs"] = sorted(ref for ref in event.get("session_refs", set()) if ref != "unknown")[:6]
        selected.append(event)
    return selected


def event_level(event: dict) -> str:
    if event["count"] >= 4 or event["source_count"] >= 4:
        return "3"
    if event["count"] >= 2 or event["source_count"] >= 2:
        return "2"
    return "1-2"


def event_source(event: dict) -> str:
    sources = []
    if event["source_count"]:
        sources.append("baseline")
    if event["count"]:
        sources.append("corpus")
    return " and ".join(sources) if sources else "corpus"


def evidence_label(event: dict) -> str:
    pieces = []
    if event["count"]:
        pieces.append(f"{event['count']} matching user lines")
    if event.get("session_refs"):
        pieces.append(f"{len(event['session_refs'])} sessions")
    if event["source_count"]:
        pieces.append(f"{event['source_count']} baseline sources")
    return "; ".join(pieces) if pieces else "0 matching user lines"


def latest_prior_report(personal: pathlib.Path) -> pathlib.Path | None:
    reports = personal / "reports" / "agent-learning"
    if not reports.exists():
        return None
    candidates = sorted(path for path in reports.glob("*.md") if path.name != "latest-approved-gates.md")
    return candidates[-1] if candidates else None


def prior_levels(report: str) -> dict[str, str]:
    levels: dict[str, str] = {}
    current_domain: str | None = None
    for line in report.splitlines():
        domain_match = re.match(r"^###\s+domain:\s*(.+?)\s*$", line, re.I)
        if domain_match:
            current_domain = domain_match.group(1).strip()
            continue
        level_match = re.match(r"^\s*-\s+(?:\*\*)?level:(?:\*\*)?\s*(.+?)\s*$", line, re.I)
        if current_domain and level_match:
            levels[current_domain] = level_match.group(1).strip()
    return levels


def current_levels(rows: list[dict]) -> dict[str, str]:
    return {row["domain"]: event_level(row) for row in rows}


def prior_report_context(personal: pathlib.Path, rows: list[dict]) -> tuple[pathlib.Path | None, list[str]]:
    prior = latest_prior_report(personal)
    if not prior:
        return None, []
    try:
        prior_text = prior.read_text(encoding="utf-8")
    except OSError:
        return prior, []
    previous = prior_levels(prior_text)
    current = current_levels(rows)
    changes = []
    for domain, level in sorted(current.items()):
        old = previous.get(domain)
        if old and old != level:
            changes.append(f"{domain}: {old} -> {level}")
    return prior, changes


def evergreen_proposals(personal: pathlib.Path, baseline: dict, corpus_user_lines: list[str]) -> list[str]:
    preferences = personal / "preferences.md"
    if not preferences.exists():
        return []
    try:
        text = preferences.read_text(encoding="utf-8")
    except OSError:
        return []
    commands = " ".join(baseline.get("validation_commands", []))
    pnpm_count = len(re.findall(r"\bpnpm\b", "\n".join(corpus_user_lines) + "\n" + commands, re.I))
    yarn_pref = re.search(r"\byarn\b", text, re.I)
    if pnpm_count < 2 or not yarn_pref:
        return []
    return [
        "- file: preferences.md",
        "  action: propose-only",
        f"  evidence_count: {pnpm_count} pnpm signals",
        "  rationale: Current repo/session evidence favors pnpm while evergreen preferences mention Yarn.",
        "  diff: Replace stale Yarn preference with a pnpm preference, after human approval.",
    ]


def first_gate(report: str) -> str:
    match = re.search(r"- gate: (.+)", report)
    return match.group(1).strip() if match else "turn recurring session evidence into concrete future-agent gates"


def skill_map_from_baseline(baseline: dict) -> dict:
    skills = []
    for path in baseline.get("skills", []):
        parts = pathlib.PurePath(path).parts
        name = parts[-2] if len(parts) >= 2 and parts[-1] == "SKILL.md" else pathlib.PurePath(path).stem
        skills.append({"name": name, "path": path, "valid": True})
    return {"repo": baseline.get("repo", "unknown"), "skills": skills, "invalid": []}


def render_skill_sections(skill_map: dict, skill_usage: dict, skill_impact: dict) -> list[str]:
    lines: list[str] = []
    skills = skill_map.get("skills", [])
    valid_skills = [item for item in skills if item.get("valid", True)]
    invalid = skill_map.get("invalid", [])
    missed = skill_usage.get("missed", [])
    failed = skill_usage.get("failed", [])
    impact_rows = skill_impact.get("skills", [])

    lines.extend(["", "## skill_inventory"])
    lines.append(f"- [skill_inventory] available_skills: {len(valid_skills)} source: skill-map input")
    if invalid:
        lines.append(f"- [skill_inventory] invalid_skills: {len(invalid)} source: skill-map input")
    for item in valid_skills[:12]:
        lines.append(f"- [skill_inventory] skill: {item.get('name', 'unknown')} source: {item.get('path', 'skill-map input')}")
    if len(valid_skills) > 12:
        lines.append(f"- [skill_inventory] omitted_skills: {len(valid_skills) - 12} source: skill-map input")

    lines.extend(["", "## skill_usage"])
    lines.append(f"- [skill_usage] expected: {', '.join(skill_usage.get('expected', [])) or 'none'} count: {len(skill_usage.get('expected', []))} source: skill-usage input")
    lines.append(f"- [skill_usage] loaded: {', '.join(skill_usage.get('loaded', [])) or 'none'} count: {len(skill_usage.get('loaded', []))} source: skill-usage input")
    lines.append(f"- [skill_usage] applied: {', '.join(skill_usage.get('applied', [])) or 'none'} count: {len(skill_usage.get('applied', []))} source: skill-usage input")

    lines.extend(["", "## skill_health"])
    if not invalid and not missed and not failed:
        lines.append("- [skill_health] no skill-health alerts found. source: skill inputs; count: 0; verify: rerun map_active_skills.py and extract_skill_usage.py.")
    for item in invalid:
        lines.append(
            f"- [skill_health] invalid: {item.get('path') or item.get('name', 'unknown')} "
            "source: skill-map input; count: 1; verify: inspect SKILL.md frontmatter and resources."
        )
    for skill in missed:
        lines.append(
            f"- [skill_health] missed_expected_skill: {skill} "
            "source: skill-usage input; count: 1; verify: rerun evaluate_skill_routing.py for the matching scope."
        )
    for skill in failed:
        lines.append(
            f"- [skill_health] loaded_but_not_applied: {skill} "
            "source: skill-usage input; count: 1; verify: inspect session closeout evidence before changing the skill."
        )

    lines.extend(["", "## skill_compensation"])
    if not impact_rows:
        lines.append("- [skill_compensation] no candidate skill adjustments. source: skill-impact input; count: 0; verify: rerun evaluate_skill_impact.py.")
    for row in impact_rows:
        candidate = row.get("candidate_adjustment")
        if not candidate:
            continue
        count = row.get("expected_sessions", row.get("missed_sessions", 1))
        lines.append(
            f"- [skill_compensation] category: candidate_skill_adjustment; skill: {row.get('skill', 'unknown')}; "
            f"signal: {row.get('impact_signal', 'needs_review')}; gate: {candidate} "
            f"source: skill-impact input; count: {count}; verify: rerun evaluate_skill_impact.py after the next session batch."
        )
    return lines


def render_report(
    corpus: str,
    baseline: dict,
    mode: str,
    personal: pathlib.Path | None = None,
    skill_map: dict | None = None,
    skill_usage: dict | None = None,
    skill_impact: dict | None = None,
    domain_rules: list[dict] | None = None,
    domain_rules_source: pathlib.Path | None = None,
) -> str:
    domain_rules = domain_rules or load_domain_rules(packaged_domain_rules_path(DEFAULT_DOMAIN_PRESET))
    today = dt.date.today().isoformat()
    user_messages = clean_user_messages(corpus)
    u_lines = [message.text for message in user_messages]
    a_lines = assistant_lines(corpus)
    corpus_meta = meta_lines(corpus)
    evidence = find_preference_evidence(u_lines, domain_rules)
    checked_sources = baseline.get("source_files", []) + baseline.get("skills", [])
    source_evidence = baseline.get("source_evidence") or [
        {"fact": f"`{path}` exists as a repo source-of-truth file.", "source": f"{path}:1"}
        for path in baseline.get("source_files", [])
    ]
    instruction_evidence = baseline.get("instruction_evidence", [])
    skill_evidence = baseline.get("skill_evidence") or [
        {"path": path, "source": f"{path}:1"} for path in baseline.get("skills", [])
    ]
    validation_evidence = baseline.get("validation_evidence") or [
        {"command": command, "source": "package.json:1"} for command in baseline.get("validation_commands", [])
    ]
    map_rows = classify_events(baseline, user_messages, domain_rules)
    if personal:
        # Drop rows for domains the operator has muted via the dashboard.
        muted = _load_muted_domains(personal)
        if muted:
            map_rows = [row for row in map_rows if row.get("domain") not in muted]
        prior_report, level_changes = prior_report_context(personal, map_rows)
        proposed_evergreen = evergreen_proposals(personal, baseline, u_lines)
    else:
        prior_report = None
        level_changes = []
        proposed_evergreen = []

    lines = [
        "# Agent Learning Report",
        "",
        f"date: {today}",
        f"mode: {mode}",
        f"repo: {baseline.get('repo', 'unknown')}",
        "",
        "## confirmed_current",
    ]
    if domain_rules_source:
        lines.append(
            f"- [confirmed_current] Domain rules loaded: {len(domain_rules)} rules. source: {domain_rules_source}"
        )
    for item in source_evidence:
        lines.append(f"- [confirmed_current] {item['fact']} source: {item['source']}")
    for bucket in ("purpose_evidence", "entrypoint_evidence", "planning_evidence", "stack_evidence", "gotcha_evidence"):
        for item in baseline.get(bucket, []):
            lines.append(f"- [confirmed_current] {item['fact']} source: {item['source']}")
    for item in instruction_evidence[:30]:
        lines.append(f"- [confirmed_current] {item['fact']} source: {item['source']}")
    if len(instruction_evidence) > 30:
        lines.append(
            f"- [confirmed_current] {len(instruction_evidence) - 30} additional instruction rules were omitted from this compact section. "
            "source: build_repo_baseline.py output"
        )
    for item in skill_evidence:
        lines.append(f"- [confirmed_current] Repo-local skill `{item['path']}` is available. source: {item['source']}")
    for item in validation_evidence:
        script = f" from `{item['script']}`" if item.get("script") else ""
        lines.append(f"- [confirmed_current] Validation command `{item['command']}` discovered{script}. source: {item['source']}")
    if prior_report:
        lines.append(f"- [confirmed_current] Prior agent-learning report found for continuity. source: {prior_report}")
    for item in corpus_meta:
        lines.append(f"- [confirmed_current] Session extraction metadata: {item}. source: corpus metadata")
    if not checked_sources and not baseline.get("validation_commands"):
        lines.append("- [confirmed_current] No repo baseline sources were discovered. source: build_repo_baseline.py output")

    lines.extend(["", "## memory_derived"])
    if map_rows:
        for row in map_rows:
            quote_part = f' quote: "{row["quotes"][0]}"' if row["quotes"] else ""
            refs_part = f"; session_refs: {', '.join(row['session_refs'])}" if row.get("session_refs") else ""
            lines.append(
                f"- [memory_derived] domain: {row['domain']}; evidence: {evidence_label(row)}; "
                f"failure_signal: {row['failure_signal']}.{quote_part}{refs_part} source: {event_source(row)}"
            )
        for change in level_changes:
            lines.append(f"- [memory_derived] level_change: {change}. source: {prior_report}")
    elif evidence:
        for item in evidence:
            lines.append(f'- [memory_derived] Session steering evidence. quote: "{quote(item)}" source: corpus')
    else:
        lines.append("- [memory_derived] No reusable user-steering evidence found in this corpus. source: corpus")

    lines.extend(["", "## needs_verification"])
    if a_lines:
        lines.append(
            "- [needs_verification] Assistant claims in transcripts are not treated as current repo truth without live checks. "
            "verify: rerun current repo or runtime checks before reuse."
        )
    else:
        lines.append("- [needs_verification] No assistant claims found. verify: rerun extraction if this is unexpected.")

    lines.extend(["", "## agent_compensation"])
    for row in map_rows:
        lines.append(f"### domain: {row['domain']}")
        lines.append(f"- marker: agent_compensation")
        lines.append(f"- level: {event_level(row)}")
        lines.append(f"- matching_lines: {row['count']}")
        if row["source_count"]:
            lines.append(f"- baseline_sources: {row['source_count']}")
        if row.get("session_refs"):
            lines.append(f"- session_refs: {', '.join(row['session_refs'])}")
        lines.append(f"- failure_signal: {row['failure_signal']} source: {event_source(row)}")
        lines.append(f"- gate_category: {row['category']}")
        lines.append(f"- gate: {row['gate']}")
        if row["quotes"]:
            lines.append(f'- quote: "{row["quotes"][0]}"')
    if not map_rows:
        lines.append("- Use read-only exploration before recommendations. source: empty baseline")

    lines.extend(["", "## self_healing_loop"])
    if map_rows:
        top = map_rows[0]
        lines.append(
            "- failure_signal -> candidate_gate -> validation_status -> next_session_load. "
            f"source: {event_source(top)}"
        )
        lines.append(
            f"- strongest_current_gate: {top['domain']} requires `{top['category']}` before action. "
            f"source: {event_source(top)}"
        )
    else:
        lines.append("- No repeated failure signal found; keep dry-run report advisory. source: corpus")

    skill_map = skill_map or skill_map_from_baseline(baseline)
    skill_usage = skill_usage or {}
    skill_impact = skill_impact or {}
    lines.extend(render_skill_sections(skill_map, skill_usage, skill_impact))

    lines.extend(["", "## next_agent_brief"])
    lines.append("- Start by reading repo-local instruction files and skill inventory.")
    lines.append("- Treat transcript-derived memories as advisory until verified in the current checkout.")
    if map_rows:
        for row in map_rows[:5]:
            lines.append(f"- {row['domain']}: {row['gate']}")
    else:
        lines.append("- Convert repeated corrections into gates, not personality claims.")
    lines.append("- Do not append durable personalization unless validation passes and --write was requested.")
    if proposed_evergreen:
        lines.extend(["", "## proposed_evergreen_diffs", *proposed_evergreen])
    return "\n".join(lines) + "\n"


def first_report_quote(report: str) -> str:
    match = re.search(r'quote: "([^"]+)"', report)
    return match.group(1) if match else "baseline"


def insert_dated_entry(path: pathlib.Path, entry: str) -> bool:
    text = path.read_text(encoding="utf-8")
    if entry in text:
        return False
    lines = text.splitlines()
    index = 0
    for idx, line in enumerate(lines):
        if re.match(r"^\[20\d\d-\d\d-\d\d\]", line):
            index = idx
            break
    lines.insert(index, entry)
    path.write_text("\n".join(lines) + "\n", encoding="utf-8")
    return True


def archive_report(personal: pathlib.Path, report: str, today: str) -> pathlib.Path:
    target_dir = personal / "reports" / "agent-learning"
    target_dir.mkdir(parents=True, exist_ok=True)
    target = target_dir / f"{today}.md"
    if target.exists() and target.read_text(encoding="utf-8") != report:
        stamp = dt.datetime.now(dt.timezone.utc).strftime("%H%M%S")
        target = target_dir / f"{today}-{stamp}.md"
    target.write_text(report, encoding="utf-8")
    return target


def archive_html_report(personal: pathlib.Path, html_text: str, today: str) -> pathlib.Path:
    target_dir = personal / "reports" / "agent-learning"
    target_dir.mkdir(parents=True, exist_ok=True)
    target = target_dir / f"{today}.html"
    if target.exists() and target.read_text(encoding="utf-8") != html_text:
        stamp = dt.datetime.now(dt.timezone.utc).strftime("%H%M%S")
        target = target_dir / f"{today}-{stamp}.html"
    target.write_text(html_text, encoding="utf-8")
    latest = target_dir / "latest-report.html"
    latest.write_text(html_text, encoding="utf-8")
    return target


def _load_muted_domains(personal: pathlib.Path) -> set[str]:
    """Read the dashboard's muted-domains.json. Safe on missing/invalid file."""
    path = personal / "actions" / "muted-domains.json"
    if not path.is_file():
        return set()
    try:
        data = json.loads(path.read_text(encoding="utf-8"))
    except (OSError, json.JSONDecodeError):
        return set()
    if not isinstance(data, list):
        return set()
    return {
        (item.get("domain") or "").strip()
        for item in data
        if isinstance(item, dict) and (item.get("domain") or "").strip()
    }


def derive_html_output(markdown_output: pathlib.Path, override: str | None) -> pathlib.Path:
    if override:
        return pathlib.Path(override)
    if markdown_output.suffix.lower() == ".md":
        return markdown_output.with_suffix(".html")
    return markdown_output.with_name(markdown_output.name + ".html")


def append_metrics_row(personal: pathlib.Path, payload: dict) -> pathlib.Path:
    """Append a single JSON record per --write run to metrics.jsonl.

    Schema (stable; new keys may be added, existing keys retain meaning):
      ts             ISO-8601 UTC instant of the run
      date           local YYYY-MM-DD
      mode           distill mode (typically "all")
      repo           baseline.repo value, or null
      totals         payload["totals"] verbatim
      by_level       {"3": N, "2": N, "1-2": N}
      by_domain      [{domain, level, count, sessions}, ...]
      domain_rules   {count, source}
      prior_report   path string or null
    """
    target_dir = personal / "reports" / "agent-learning"
    target_dir.mkdir(parents=True, exist_ok=True)
    metrics_path = target_dir / "metrics.jsonl"

    rows = payload.get("agent_compensation", {}).get("rows", []) or []
    by_level: dict[str, int] = {}
    for row in rows:
        level = row.get("level", "1-2")
        by_level[level] = by_level.get(level, 0) + 1
    by_domain = [
        {
            "domain": row["domain"],
            "level": row.get("level", "1-2"),
            "count": row.get("count", 0),
            "sessions": len(row.get("session_refs", []) or []),
            "gate_category": row.get("gate_category", ""),
        }
        for row in rows
    ]
    record = {
        "ts": dt.datetime.now(dt.timezone.utc).isoformat(timespec="seconds"),
        "date": payload.get("date"),
        "mode": payload.get("mode"),
        "repo": payload.get("repo"),
        "totals": payload.get("totals", {}),
        "by_level": by_level,
        "by_domain": by_domain,
        "domain_rules": payload.get("domain_rules", {}),
        "prior_report": payload.get("prior_report_path"),
    }
    with metrics_path.open("a", encoding="utf-8") as fh:
        fh.write(json.dumps(record, ensure_ascii=False) + "\n")
    return metrics_path


def write_personal_updates(personal: pathlib.Path, report: str, today: str) -> list[pathlib.Path]:
    errors = validate(report)
    if errors:
        raise ValueError("; ".join(errors))

    insights = personal / "insights.md"
    learning = personal / "learning.md"
    if not insights.exists() or not learning.exists():
        raise FileNotFoundError("personal insights.md and learning.md must exist before --write")

    source_quote = first_report_quote(report)
    touched = [archive_report(personal, report, today)]
    gate = first_gate(report)
    insight_entry = (
        f'[{today}] Agent-learning run found reusable session steering that future agents should '
        f'turn into concrete gates. Top gate: {gate} (source: "{source_quote}")'
    )
    if insert_dated_entry(insights, insight_entry):
        touched.append(insights)

    # One durable line per gate at level >= 2. Idempotent — re-runs on the
    # same day with the same gate text are skipped by insert_dated_entry.
    # The older "summary blurb" line was dropped — it duplicated across runs
    # when the top quote varied without adding signal beyond `insights.md`.
    learning_appended = False
    for parsed in parse_gates(report):
        level = (parsed.level or "").strip()
        if level not in {"2", "3"}:
            continue
        evidence_bit = ""
        if parsed.evidence_count and parsed.evidence_unit:
            evidence_bit = f" ({parsed.evidence_count} {parsed.evidence_unit})"
        gate_entry = (
            f"[{today}] [{parsed.domain}] level {level} · {parsed.gate_category}: "
            f"{parsed.gate}{evidence_bit}"
        )
        if insert_dated_entry(learning, gate_entry):
            learning_appended = True
    if learning_appended:
        touched.append(learning)

    return touched


def export_approved_gates(report_path: pathlib.Path, report: str, output: pathlib.Path, max_domains: int) -> pathlib.Path:
    errors = validate(report)
    if errors:
        raise ValueError("; ".join(errors))
    gates = parse_gates(report)
    if not gates:
        raise ValueError("no agent_compensation gates found")
    output.parent.mkdir(parents=True, exist_ok=True)
    output.write_text(render_registry(report_path.resolve(), gates, max_domains), encoding="utf-8")
    return output


def resolve_personal_arg(value: str | None) -> pathlib.Path | None:
    if value:
        return pathlib.Path(value).expanduser()
    env_value = os.environ.get("AGENT_LEARNING_PERSONAL")
    if env_value:
        return pathlib.Path(env_value).expanduser()
    return None


def main(argv: list[str] | None = None) -> int:
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument("--corpus", required=True)
    parser.add_argument("--baseline", required=True)
    parser.add_argument("--output", required=True)
    parser.add_argument("--mode", default="all")
    parser.add_argument("--write", action="store_true")
    parser.add_argument("--personal", help="Personal memory root. Required for --write unless AGENT_LEARNING_PERSONAL is set.")
    parser.add_argument(
        "--approved-gates-output",
        help="Optional markdown path for compact approved gates registry. With --write, defaults to personal/reports/agent-learning/latest-approved-gates.md.",
    )
    parser.add_argument("--skill-map", help="Optional active skill map JSON from map_active_skills.py.")
    parser.add_argument("--skill-usage", help="Optional skill usage JSON from extract_skill_usage.py.")
    parser.add_argument("--skill-impact", help="Optional skill impact JSON from evaluate_skill_impact.py.")
    parser.add_argument("--domain-rules", help="Optional active domain-rules JSON. Defaults to repo config, env, then the generic preset.")
    parser.add_argument("--domain-preset", default=DEFAULT_DOMAIN_PRESET, help="Packaged domain-rules preset to use when no active rules file is configured.")
    parser.add_argument(
        "--skill-context-output",
        help="Optional latest-skill-context.md output path. With --write, defaults to personal/reports/agent-learning/latest-skill-context.md.",
    )
    parser.add_argument("--max-gate-domains", type=int, default=9)
    parser.add_argument(
        "--html-output",
        help="Optional explicit HTML report path. Defaults to <output> with .html extension.",
    )
    parser.add_argument(
        "--no-html",
        action="store_true",
        help="Skip HTML report generation. Markdown remains the canonical output.",
    )
    args = parser.parse_args(argv)
    if args.max_gate_domains < 1:
        parser.error("--max-gate-domains must be at least 1")

    corpus = pathlib.Path(args.corpus).read_text(encoding="utf-8")
    baseline = load_json(pathlib.Path(args.baseline))
    personal = resolve_personal_arg(args.personal)
    if args.write and personal is None:
        print("--write requires --personal or AGENT_LEARNING_PERSONAL", file=sys.stderr)
        return 1
    skill_map = load_optional_json(args.skill_map) or skill_map_from_baseline(baseline)
    skill_usage = load_optional_json(args.skill_usage)
    skill_impact = load_optional_json(args.skill_impact)
    try:
        domain_rules, domain_rules_source = resolve_domain_rules(baseline, args.domain_rules, args.domain_preset)
    except ValueError as error:
        print(str(error), file=sys.stderr)
        return 1
    report = render_report(
        corpus,
        baseline,
        args.mode,
        personal,
        skill_map,
        skill_usage,
        skill_impact,
        domain_rules,
        domain_rules_source,
    )
    output_path = pathlib.Path(args.output)
    output_path.write_text(report, encoding="utf-8")

    payload = None
    html_text: str | None = None
    html_path: pathlib.Path | None = None
    if not args.no_html or args.write:
        from render_html_report import build_report_payload, render_html_report

        payload = build_report_payload(
            corpus,
            baseline,
            args.mode,
            personal,
            skill_map,
            skill_usage,
            skill_impact,
            domain_rules,
            domain_rules_source,
        )
        if not args.no_html:
            html_text = render_html_report(payload)
            html_path = derive_html_output(output_path, args.html_output)
            html_path.parent.mkdir(parents=True, exist_ok=True)
            html_path.write_text(html_text, encoding="utf-8")

    if args.write:
        today = dt.date.today().isoformat()
        touched = write_personal_updates(personal, report, today)
        gates_output = pathlib.Path(args.approved_gates_output) if args.approved_gates_output else (
            personal / "reports" / "agent-learning" / "latest-approved-gates.md"
        )
        touched.append(export_approved_gates(touched[0], report, gates_output, args.max_gate_domains))
        context_output = pathlib.Path(args.skill_context_output) if args.skill_context_output else (
            personal / "reports" / "agent-learning" / "latest-skill-context.md"
        )
        touched.append(write_context(context_output, skill_map, skill_usage, skill_impact))
        if html_text is not None:
            touched.append(archive_html_report(personal, html_text, today))
        if payload is not None:
            touched.append(append_metrics_row(personal, payload))
            # Best-effort dashboard render. The dashboard bundle is built once
            # via `pnpm build` under dashboard/web; if it's missing we just log
            # and continue — markdown + HTML report are still authoritative.
            try:
                from render_dashboard import DEFAULT_BUNDLE, render as render_dashboard

                if DEFAULT_BUNDLE.is_file():
                    touched.append(render_dashboard(personal, DEFAULT_BUNDLE, history_limit=180))
            except Exception as error:  # noqa: BLE001
                print(f"distill: dashboard render skipped ({error})", file=sys.stderr)
        print("append-only updates: written " + ", ".join(str(path) for path in touched))
    else:
        refused = False
        personal_root = pathlib.Path(args.personal) if args.personal else None
        for flag, value in (
            ("--approved-gates-output", args.approved_gates_output),
            ("--skill-context-output", args.skill_context_output),
        ):
            if not value:
                continue
            refused = True
            target = pathlib.Path(value)
            inside_personal = False
            if personal_root is not None:
                try:
                    target.resolve().relative_to(personal_root.resolve())
                    inside_personal = True
                except ValueError:
                    inside_personal = False
            if inside_personal:
                print(
                    f"{flag} inside --personal requires --write; pass --write to persist",
                    file=sys.stderr,
                )
            else:
                print(
                    f"distill: refusing to write {flag} without --write; pass --write to persist",
                    file=sys.stderr,
                )
        if refused:
            return 1
        suffix = f"; html: {html_path}" if html_path else ""
        print(f"append-only updates: dry-run{suffix}")
    return 0


if __name__ == "__main__":
    raise SystemExit(main())
