#!/usr/bin/env python3
# -*- coding: utf-8 -*-

"""
Pre-normalization Unicode scanner for ornamentation Excel files.

Purpose
-------
Read one or more ornamentation .xlsx files, inspect the text column only,
and produce a forensic report of every unique character encountered BEFORE
normalization. Directory mode searches recursively so future Excel files can be
placed in subfolders without changing this script.

This script is READ-ONLY. It never modifies the source workbooks.

Assumed workbook shape
----------------------
- First worksheet only
- Row 1 = header
- Column 1 = sura
- Column 2 = verse
- Column 3 = text
- Ignore all remaining columns

Usage
-----
python ornamentation_scanner.py --in-dir RAW_XLSX --out-dir REPORTS
python ornamentation_scanner.py --in-file "Hafs_Ornamentation.xlsx" --out-dir REPORTS
"""

from __future__ import annotations

import argparse
import json
import os
import re
import unicodedata
from collections import Counter, defaultdict
from pathlib import Path
from typing import Dict, Iterable, List, Tuple

from openpyxl import load_workbook


# -----------------------------
# Canonical output contract
# -----------------------------
CANONICAL_28 = set("ابتثجحخدذرزسشصضطظعغفقكلمنهوي")

YEH_VARIANTS = {"ی", "ے", "ۓ", "ێ", "ۍ", "ى", "ئ"}
ALEF_VARIANTS = {"أ", "إ", "آ", "ٱ"}
HAMZA_FAMILY = {"ء", "ؤ", "ئ"}
TA_VARIANTS = {"ة"}

ARABIC_INDIC_DIGITS = set("٠١٢٣٤٥٦٧٨٩")
ASCII_DIGITS = set("0123456789")

KNOWN_WHITESPACE = {
    " ", "\t", "\n", "\r", "\u00A0", "\u2000", "\u2001", "\u2002",
    "\u2003", "\u2004", "\u2005", "\u2006", "\u2007", "\u2008",
    "\u2009", "\u200A", "\u202F", "\u205F", "\u3000",
}

INVISIBLE_FORMATTING = {
    "\ufeff", "\u200b", "\u200c", "\u200d", "\u200e", "\u200f",
    "\u202a", "\u202b", "\u202c", "\u202d", "\u202e", "\u2066",
    "\u2067", "\u2068", "\u2069",
}

RE_ORNAMENTATION_TOKEN = re.compile(r"(?i)(^|[\s_\-]+)ornamentation(?=$|[\s_\-]+)")
RE_MULTI_UNDERSCORES = re.compile(r"_+")
RE_VARIANT_SUFFIX = re.compile(r"(\([^)]+\))$")


def ulabel(ch: str) -> str:
    return f"U+{ord(ch):04X} {unicodedata.name(ch, '<no name>')} [{unicodedata.category(ch)}]"


def output_stem_from_path(path: Path) -> str:
    stem = path.stem
    # Extract trailing (variant) suffix e.g. (Medina I) before cleaning
    m = RE_VARIANT_SUFFIX.search(stem)
    variant = m.group(1) if m else ""
    if variant:
        stem = stem[:m.start()]
    stem = RE_ORNAMENTATION_TOKEN.sub(" ", stem)
    stem = stem.replace("-", " ").replace("_", " ")
    stem = " ".join(stem.split())
    stem = stem.replace(" ", "_")
    stem = RE_MULTI_UNDERSCORES.sub("_", stem).strip("_")
    stem = stem or "output"
    if variant:
        stem = stem + variant
    return stem


def classify_char(ch: str) -> str:
    if ch in CANONICAL_28:
        return "canonical_letters"
    if ch in YEH_VARIANTS:
        return "yeh_variants"
    if ch in ALEF_VARIANTS:
        return "alef_variants"
    if ch in HAMZA_FAMILY:
        return "hamza_family"
    if ch in TA_VARIANTS:
        return "ta_variants"
    if ch in ARABIC_INDIC_DIGITS:
        return "arabic_indic_digits"
    if ch in ASCII_DIGITS:
        return "ascii_digits"
    if ch in INVISIBLE_FORMATTING:
        return "invisible_formatting"
    if ch in KNOWN_WHITESPACE or ch.isspace():
        return "whitespace"

    cat = unicodedata.category(ch)
    if cat in {"Mn", "Mc", "Me"}:
        return "combining_marks"

    blockish = ord(ch)
    if 0x0600 <= blockish <= 0x06FF or 0x0750 <= blockish <= 0x077F or 0x08A0 <= blockish <= 0x08FF:
        return "other_arabic_chars"

    return "other_unknown_chars"


def header_is_valid(a, b, c) -> bool:
    a = str(a).strip().lower() if a is not None else ""
    b = str(b).strip().lower() if b is not None else ""
    c = str(c).strip().lower() if c is not None else ""
    return a == "sura" and b == "verse" and c in {"text", "arabic text"}


def scan_workbook(path: Path) -> dict:
    wb = load_workbook(path, read_only=True, data_only=True)
    ws = wb[wb.sheetnames[0]]

    header = next(ws.iter_rows(min_row=1, max_row=1, values_only=True))
    if not header_is_valid(*(header[:3] if len(header) >= 3 else (None, None, None))):
        raise ValueError(f"{path.name}: unexpected header in first 3 columns: {header[:3]}")

    category_counts = defaultdict(Counter)
    row_examples: Dict[str, List[dict]] = defaultdict(list)
    row_count = 0

    for row_idx, row in enumerate(ws.iter_rows(min_row=2, values_only=True), start=2):
        sura = row[0] if len(row) > 0 else None
        verse = row[1] if len(row) > 1 else None
        text = row[2] if len(row) > 2 else None

        if sura is None and verse is None and (text is None or str(text).strip() == ""):
            continue

        row_count += 1
        txt = "" if text is None else str(text)

        for ch in txt:
            category = classify_char(ch)
            category_counts[category][ulabel(ch)] += 1

            if len(row_examples[ulabel(ch)]) < 3:
                row_examples[ulabel(ch)].append({
                    "row": row_idx,
                    "sura": sura,
                    "verse": verse,
                    "text_excerpt": txt[:120]
                })

    summary = {
        "file": path.name,
        "sheet": ws.title,
        "rows_scanned": row_count,
        "categories": {cat: dict(counter.most_common()) for cat, counter in category_counts.items()},
        "examples": row_examples,
    }
    return summary


def write_reports(summary: dict, out_dir: Path) -> None:
    out_dir.mkdir(parents=True, exist_ok=True)
    stem = output_stem_from_path(Path(summary["file"]))

    json_path = out_dir / f"{stem}_unicode_report.json"
    txt_path = out_dir / f"{stem}_unicode_report.txt"

    with open(json_path, "w", encoding="utf-8") as f:
        json.dump(summary, f, ensure_ascii=False, indent=2)

    lines = []
    lines.append(f"FILE: {summary['file']}")
    lines.append(f"SHEET: {summary['sheet']}")
    lines.append(f"ROWS SCANNED: {summary['rows_scanned']}")
    lines.append("")

    for category in sorted(summary["categories"].keys()):
        lines.append(f"CATEGORY: {category}")
        counts = summary["categories"][category]
        if not counts:
            lines.append("  (none)")
        else:
            for label, count in counts.items():
                lines.append(f"  {label} :: {count}")
        lines.append("")

    with open(txt_path, "w", encoding="utf-8") as f:
        f.write("\n".join(lines))


DEFAULT_EXCLUDED_DIRS = {
    ".git", ".venv", "venv", "__pycache__",
    "unicode_reports", "normalized_outputs_all_with_json",
}


def is_excluded_excel_path(path: Path, root: Path, extra_excluded_dirs: set[str] | None = None) -> bool:
    """Return True for generated/temp files that should not be treated as inputs."""
    excluded = set(DEFAULT_EXCLUDED_DIRS)
    if extra_excluded_dirs:
        excluded.update(extra_excluded_dirs)

    # Skip Excel temp/lock files.
    if path.name.startswith("~$"):
        return True

    try:
        rel_parts = path.relative_to(root).parts
    except ValueError:
        rel_parts = path.parts

    return any(part in excluded for part in rel_parts[:-1])


def collect_input_files(in_file: str | None, in_dir: str | None, extra_excluded_dirs: set[str] | None = None) -> List[Path]:
    files: List[Path] = []
    root = Path(in_dir).resolve() if in_dir else Path.cwd().resolve()

    if in_file:
        files.append(Path(in_file))

    if in_dir:
        # Recursive discovery: find Excel files anywhere under the input directory.
        files.extend(sorted(root.rglob("*.xlsx")))

    files = [p for p in files if p.exists() and not is_excluded_excel_path(p.resolve(), root, extra_excluded_dirs)]

    if not files:
        raise RuntimeError("No input .xlsx files found.")

    # deduplicate while preserving order
    seen = set()
    uniq = []
    for p in files:
        resolved = p.resolve()
        if resolved not in seen:
            uniq.append(resolved)
            seen.add(resolved)
    return uniq


def main() -> None:
    ap = argparse.ArgumentParser(description="Scan ornamentation Excel files for all Unicode characters present.")
    ap.add_argument("--in-file", help="Single input .xlsx file")
    ap.add_argument("--in-dir", help="Directory of input .xlsx files")
    ap.add_argument("--out-dir", required=True, help="Directory for output reports")
    args = ap.parse_args()

    out_dir = Path(args.out_dir)
    extra_excluded_dirs = {out_dir.name}
    files = collect_input_files(args.in_file, args.in_dir, extra_excluded_dirs=extra_excluded_dirs)

    for path in files:
        summary = scan_workbook(path)
        write_reports(summary, out_dir)
        print(f"[OK] scanned {path.name}")


if __name__ == "__main__":
    main()
