"""Замер автонормализации: извлечь весь архив, сопоставить со справочником, дать %."""
import sys
from collections import Counter
from pathlib import Path

REPO = Path(__file__).resolve().parents[2]
sys.path.insert(0, str(REPO))

from sentence_transformers import SentenceTransformer

from etl.dictionary import load_dictionary
from etl.extractors import extract
from etl.normalize import Matcher

# --- собрать все извлечённые позиции ---
rows: list[tuple[str, str | None]] = []
for path in sorted((REPO / "data/raw").glob("*")):
    for row in extract(str(path)).rows:
        rows.append((row.service_name_raw, row.service_code_source))
print(f"позиций всего: {len(rows)}")

# дедуп по (имя, код) — одинаковые строки сопоставляем один раз
occurrences: Counter[tuple[str, str | None]] = Counter(rows)
keys = list(occurrences)
print(f"уникальных (имя, код): {len(keys)}")

services = load_dictionary(REPO / "data/reference/dictionary.xlsx")
print("загружаю модель эмбеддингов (первый раз — скачивание)…")
embedder = SentenceTransformer("paraphrase-multilingual-MiniLM-L12-v2")
matcher = Matcher(services, embedder=embedder)

print("сопоставляю…")
results = matcher.match_batch([k[0] for k in keys], [k[1] for k in keys])

by_method: Counter[str | None] = Counter()
total = matched = 0
samples: dict[str | None, list] = {}
canon_by_id = {s.service_id: s.name_ru for s in services}
for key, res in zip(keys, results, strict=True):
    count = occurrences[key]
    total += count
    by_method[res.method] += count
    if res.service_id:
        matched += count
    bucket = samples.setdefault(res.method, [])
    if len(bucket) < 5:
        bucket.append((key[0], canon_by_id.get(res.service_id, "—"), round(res.confidence, 2)))

print(f"\nАВТОНОРМАЛИЗАЦИЯ: {matched}/{total} = {100 * matched / total:.0f}%  (цель ТЗ ≥70%)")
for method in ("code", "exact", "embedding", "fuzzy", None):
    print(f"  {method or 'unmatched':10}: {by_method[method]} строк")

print("\nпримеры сопоставлений:")
for method in ("code", "exact", "embedding", "fuzzy"):
    for raw, canon, score in samples.get(method, [])[:3]:
        print(f"  [{method:9}] «{raw[:36]}» → «{canon[:36]}» ({score})")
print("\nпримеры unmatched:")
for raw, _canon, _score in samples.get(None, [])[:6]:
    print(f"  «{raw[:52]}»")