#!/usr/bin/env python3
"""
Validation report for 14_initial_engine outputs.

Checks:
  1. All expected output files exist (JSON, CSV, XLSX) for every mushaf
  2. Every JSON file has all 29 initialed suras
  3. JSON verse counts match CSV row counts per mushaf/sura
  4. JSON verse counts match XLSX row counts per mushaf/sura
  5. No verse has an empty text field
  6. All letter counts are non-negative integers
  7. Letter count totals from explicit columns match recount from text
  8. Sura verse lists are sorted by verse number with no duplicate verse numbers
  9. initials field in flat rows matches INITIALS_BY_SURA
 10. Byte-identical re-run (runs engine a second time, rehashes all outputs)
"""

import csv
import hashlib
import json
import subprocess
import sys
from pathlib import Path

from openpyxl import load_workbook

# ---------------------------------------------------------------------------
# Config (must match 14_initial_engine.py)
# ---------------------------------------------------------------------------

BASE_DIR = Path(__file__).resolve().parent
OUTPUT_DIR = BASE_DIR / "14_initial_engine_outputs"
JSON_DIR = OUTPUT_DIR / "json"
CSV_DIR = OUTPUT_DIR / "csv"
EXCEL_DIR = OUTPUT_DIR / "excel"

ARABIC_LETTERS = [
    "ا","ب","ت","ث","ج","ح","خ","د","ذ","ر","ز","س","ش","ص",
    "ض","ط","ظ","ع","غ","ف","ق","ك","ل","م","ن","ه","و","ي"
]

INITIALS_BY_SURA = {
    2:  ["ا","ل","م"],  3:  ["ا","ل","م"],  7:  ["ا","ل","م","ص"],
    10: ["ا","ل","ر"],  11: ["ا","ل","ر"],  12: ["ا","ل","ر"],
    13: ["ا","ل","م","ر"], 14: ["ا","ل","ر"], 15: ["ا","ل","ر"],
    19: ["ك","ه","ي","ع","ص"], 20: ["ط","ه"],
    26: ["ط","س","م"],  27: ["ط","س"],       28: ["ط","س","م"],
    29: ["ا","ل","م"],  30: ["ا","ل","م"],   31: ["ا","ل","م"],
    32: ["ا","ل","م"],  36: ["ي","س"],       38: ["ص"],
    40: ["ح","م"],      41: ["ح","م"],        42: ["ح","م","ع","س","ق"],
    43: ["ح","م"],      44: ["ح","م"],        45: ["ح","م"],
    46: ["ح","م"],      50: ["ق"],            68: ["ن"],
}

EXPECTED_SURAS = sorted(INITIALS_BY_SURA.keys())

# ---------------------------------------------------------------------------
# Helpers
# ---------------------------------------------------------------------------

PASS = "PASS"
FAIL = "FAIL"
WARN = "WARN"

issues: list[tuple[str, str, str]] = []   # (level, mushaf_or_scope, message)


def record(level: str, scope: str, msg: str) -> None:
    issues.append((level, scope, msg))


def sha256_file(path: Path) -> str:
    h = hashlib.sha256()
    with open(path, "rb") as f:
        for chunk in iter(lambda: f.read(65536), b""):
            h.update(chunk)
    return h.hexdigest()


def count_letters(text: str) -> dict[str, int]:
    counts = {l: 0 for l in ARABIC_LETTERS}
    for ch in text:
        if ch in counts:
            counts[ch] += 1
    return counts


# ---------------------------------------------------------------------------
# Discover mushafs from JSON output directory
# ---------------------------------------------------------------------------

json_files = sorted(JSON_DIR.glob("*_initial_counts.json"))
MUSHAFS = [f.stem.replace("_initial_counts", "") for f in json_files]

if not MUSHAFS:
    print("ERROR: No JSON output files found. Run 14_initial_engine.py first.")
    sys.exit(1)

print(f"Mushafs found: {', '.join(MUSHAFS)}\n")
print("=" * 70)

# ---------------------------------------------------------------------------
# Check 1: All three output files exist for every mushaf
# ---------------------------------------------------------------------------

print("\n[1] Output file existence")
for mushaf in MUSHAFS:
    for fmt, directory in [("JSON", JSON_DIR), ("CSV", CSV_DIR), ("XLSX", EXCEL_DIR)]:
        suffix = ".json" if fmt == "JSON" else (".csv" if fmt == "CSV" else ".xlsx")
        p = directory / f"{mushaf}_initial_counts{suffix}"
        if p.exists():
            print(f"  OK  {fmt:4s}  {p.name}")
        else:
            print(f"  FAIL {fmt:4s}  MISSING: {p.name}")
            record(FAIL, mushaf, f"Missing {fmt} output: {p.name}")

# ---------------------------------------------------------------------------
# Check 2-8: JSON structural and data quality checks
# ---------------------------------------------------------------------------

print("\n[2-8] JSON structural and data checks")

json_verse_counts: dict[str, dict[int, int]] = {}   # mushaf -> sura -> verse_count

for mushaf in MUSHAFS:
    jpath = JSON_DIR / f"{mushaf}_initial_counts.json"
    with open(jpath, encoding="utf-8") as f:
        data = json.load(f)

    suras_present = [int(k) for k in data.get("suras", {}).keys()]
    missing_suras = [s for s in EXPECTED_SURAS if s not in suras_present]
    if missing_suras:
        record(FAIL, mushaf, f"Missing suras in JSON: {missing_suras}")
        print(f"  FAIL  {mushaf}: missing suras {missing_suras}")
    else:
        print(f"  OK    {mushaf}: all 29 suras present")

    verse_counts: dict[int, int] = {}
    for sura in EXPECTED_SURAS:
        sura_obj = data["suras"].get(str(sura), {})
        verses = sura_obj.get("verses", [])
        verse_counts[sura] = len(verses)

        expected_initials = INITIALS_BY_SURA[sura]
        actual_initials = sura_obj.get("initials", [])
        if actual_initials != expected_initials:
            record(FAIL, mushaf, f"Sura {sura}: initials mismatch — expected {expected_initials}, got {actual_initials}")

        verse_nums = [v["verse"] for v in verses]
        if verse_nums != sorted(set(verse_nums)):
            record(FAIL, mushaf, f"Sura {sura}: verse list not sorted or has duplicates")

        for v in verses:
            text = v.get("text", "")
            if not text.strip():
                record(WARN, mushaf, f"Sura {sura} verse {v['verse']}: empty text field")

            letters = v.get("letters", {})
            for letter in ARABIC_LETTERS:
                val = letters.get(letter, 0)
                if not isinstance(val, int) or val < 0:
                    record(FAIL, mushaf, f"Sura {sura} verse {v['verse']}: invalid count for '{letter}': {val!r}")

            # Check 7: recount from text and compare totals
            recount = count_letters(text)
            stored_total = sum(letters.get(l, 0) for l in ARABIC_LETTERS)
            recount_total = sum(recount.values())
            if stored_total != recount_total:
                record(
                    WARN, mushaf,
                    f"Sura {sura} verse {v['verse']}: stored letter total ({stored_total}) "
                    f"!= recount from text ({recount_total}). "
                    "Likely source uses a different normalization — not necessarily wrong."
                )

    json_verse_counts[mushaf] = verse_counts

# ---------------------------------------------------------------------------
# Check 3: JSON vs CSV verse counts
# ---------------------------------------------------------------------------

print("\n[3] JSON vs CSV row counts")

csv_verse_counts: dict[str, dict[int, int]] = {}

for mushaf in MUSHAFS:
    cpath = CSV_DIR / f"{mushaf}_initial_counts.csv"
    if not cpath.exists():
        continue
    with open(cpath, encoding="utf-8-sig", newline="") as f:
        rows = list(csv.DictReader(f))

    counts: dict[int, int] = {}
    for row in rows:
        try:
            sura = int(row.get("sura", -1))
        except (ValueError, TypeError):
            continue
        counts[sura] = counts.get(sura, 0) + 1

    csv_verse_counts[mushaf] = counts
    mismatches = []
    for sura in EXPECTED_SURAS:
        j = json_verse_counts.get(mushaf, {}).get(sura, 0)
        c = counts.get(sura, 0)
        if j != c:
            mismatches.append(f"sura {sura}: JSON={j} CSV={c}")
    if mismatches:
        record(FAIL, mushaf, "JSON/CSV verse count mismatch: " + "; ".join(mismatches))
        print(f"  FAIL  {mushaf}: {'; '.join(mismatches)}")
    else:
        total = sum(counts.get(s, 0) for s in EXPECTED_SURAS)
        print(f"  OK    {mushaf}: {total} rows match JSON")

# ---------------------------------------------------------------------------
# Check 4: JSON vs XLSX verse counts
# ---------------------------------------------------------------------------

print("\n[4] JSON vs XLSX row counts")

for mushaf in MUSHAFS:
    xpath = EXCEL_DIR / f"{mushaf}_initial_counts.xlsx"
    if not xpath.exists():
        continue
    wb = load_workbook(xpath, read_only=True, data_only=True)
    ws = wb.active
    header_row = next(ws.iter_rows(min_row=1, max_row=1, values_only=True), None)
    if header_row is None:
        wb.close()
        record(FAIL, mushaf, "XLSX has no header row")
        continue
    headers = [str(h).strip() if h is not None else "" for h in header_row]
    try:
        sura_idx = headers.index("sura")
    except ValueError:
        wb.close()
        record(FAIL, mushaf, "XLSX missing 'sura' column")
        continue

    counts: dict[int, int] = {}
    for row in ws.iter_rows(min_row=2, values_only=True):
        if row is None:
            continue
        try:
            sura = int(row[sura_idx])
        except (ValueError, TypeError, IndexError):
            continue
        counts[sura] = counts.get(sura, 0) + 1
    wb.close()

    mismatches = []
    for sura in EXPECTED_SURAS:
        j = json_verse_counts.get(mushaf, {}).get(sura, 0)
        x = counts.get(sura, 0)
        if j != x:
            mismatches.append(f"sura {sura}: JSON={j} XLSX={x}")
    if mismatches:
        record(FAIL, mushaf, "JSON/XLSX verse count mismatch: " + "; ".join(mismatches))
        print(f"  FAIL  {mushaf}: {'; '.join(mismatches)}")
    else:
        total = sum(counts.get(s, 0) for s in EXPECTED_SURAS)
        print(f"  OK    {mushaf}: {total} rows match JSON")

# ---------------------------------------------------------------------------
# Check 9: initials column in CSV matches INITIALS_BY_SURA
# ---------------------------------------------------------------------------

print("\n[9] CSV initials column correctness")

for mushaf in MUSHAFS:
    cpath = CSV_DIR / f"{mushaf}_initial_counts.csv"
    if not cpath.exists():
        continue
    with open(cpath, encoding="utf-8-sig", newline="") as f:
        rows = list(csv.DictReader(f))

    bad = []
    for row in rows:
        try:
            sura = int(row.get("sura", -1))
        except (ValueError, TypeError):
            continue
        if sura not in INITIALS_BY_SURA:
            continue
        expected = "".join(INITIALS_BY_SURA[sura])
        actual = row.get("initials", "")
        if actual != expected:
            bad.append(f"sura {sura}: expected '{expected}' got '{actual}'")
    if bad:
        for b in bad[:5]:
            record(FAIL, mushaf, f"initials mismatch — {b}")
        print(f"  FAIL  {mushaf}: {len(bad)} initials mismatch(es)")
    else:
        print(f"  OK    {mushaf}: all initials correct")

# ---------------------------------------------------------------------------
# Check 10: Byte-identical re-run
# ---------------------------------------------------------------------------

print("\n[10] Byte-identical re-run")

before_hashes = {
    p.name: sha256_file(p)
    for p in sorted(OUTPUT_DIR.rglob("*"))
    if p.is_file()
}

result = subprocess.run(
    [sys.executable, str(BASE_DIR / "14_initial_engine.py")],
    capture_output=True, text=True
)
if result.returncode != 0:
    record(FAIL, "engine", f"Engine exited with code {result.returncode}:\n{result.stderr}")
    print(f"  FAIL  Engine returned non-zero exit code {result.returncode}")
else:
    after_hashes = {
        p.name: sha256_file(p)
        for p in sorted(OUTPUT_DIR.rglob("*"))
        if p.is_file()
    }
    diffs = [
        name for name in set(before_hashes) | set(after_hashes)
        if before_hashes.get(name) != after_hashes.get(name)
    ]
    if diffs:
        for d in diffs:
            record(FAIL, "determinism", f"Hash changed on re-run: {d}")
        print(f"  FAIL  {len(diffs)} file(s) changed on re-run: {diffs}")
    else:
        print(f"  OK    All {len(after_hashes)} output files are byte-identical across two runs")

# ---------------------------------------------------------------------------
# Summary
# ---------------------------------------------------------------------------

print("\n" + "=" * 70)
print("VALIDATION SUMMARY")
print("=" * 70)

fails  = [(s, m) for l, s, m in issues if l == FAIL]
warns  = [(s, m) for l, s, m in issues if l == WARN]

if not fails and not warns:
    print(f"\n  ALL CHECKS PASSED  ({len(MUSHAFS)} mushafs, {len(EXPECTED_SURAS)} suras each)\n")
else:
    if fails:
        print(f"\n  FAILURES ({len(fails)}):")
        for scope, msg in fails:
            print(f"    [{scope}] {msg}")
    if warns:
        print(f"\n  WARNINGS ({len(warns)}):")
        for scope, msg in warns[:20]:
            print(f"    [{scope}] {msg}")
        if len(warns) > 20:
            print(f"    ... and {len(warns) - 20} more warnings")

print()
sys.exit(1 if fails else 0)
