admins
14462e8c10
- etl/normalize/matcher.py — каскад код→точное→эмбеддинги→fuzzy→unmatched - etl/validate/rules.py — правила ТЗ §4.4; проверено: 15588 done / 11 needs_review - etl/extractors/vision.py — Gemini Vision для битых/нестандартных PDF - замер % автонормализации — после установки модели эмбеддингов Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
60 lines
2.5 KiB
Python
60 lines
2.5 KiB
Python
"""Замер автонормализации: извлечь весь архив, сопоставить со справочником, дать %."""
|
|
import sys
|
|
from collections import Counter
|
|
from pathlib import Path
|
|
|
|
REPO = Path(__file__).resolve().parents[2]
|
|
sys.path.insert(0, str(REPO))
|
|
|
|
from sentence_transformers import SentenceTransformer
|
|
|
|
from etl.dictionary import load_dictionary
|
|
from etl.extractors import extract
|
|
from etl.normalize import Matcher
|
|
|
|
# --- собрать все извлечённые позиции ---
|
|
rows: list[tuple[str, str | None]] = []
|
|
for path in sorted((REPO / "data/raw").glob("*")):
|
|
for row in extract(str(path)).rows:
|
|
rows.append((row.service_name_raw, row.service_code_source))
|
|
print(f"позиций всего: {len(rows)}")
|
|
|
|
# дедуп по (имя, код) — одинаковые строки сопоставляем один раз
|
|
occurrences: Counter[tuple[str, str | None]] = Counter(rows)
|
|
keys = list(occurrences)
|
|
print(f"уникальных (имя, код): {len(keys)}")
|
|
|
|
services = load_dictionary(REPO / "data/reference/dictionary.xlsx")
|
|
print("загружаю модель эмбеддингов (первый раз — скачивание)…")
|
|
embedder = SentenceTransformer("paraphrase-multilingual-MiniLM-L12-v2")
|
|
matcher = Matcher(services, embedder=embedder)
|
|
|
|
print("сопоставляю…")
|
|
results = matcher.match_batch([k[0] for k in keys], [k[1] for k in keys])
|
|
|
|
by_method: Counter[str | None] = Counter()
|
|
total = matched = 0
|
|
samples: dict[str | None, list] = {}
|
|
canon_by_id = {s.service_id: s.name_ru for s in services}
|
|
for key, res in zip(keys, results, strict=True):
|
|
count = occurrences[key]
|
|
total += count
|
|
by_method[res.method] += count
|
|
if res.service_id:
|
|
matched += count
|
|
bucket = samples.setdefault(res.method, [])
|
|
if len(bucket) < 5:
|
|
bucket.append((key[0], canon_by_id.get(res.service_id, "—"), round(res.confidence, 2)))
|
|
|
|
print(f"\nАВТОНОРМАЛИЗАЦИЯ: {matched}/{total} = {100 * matched / total:.0f}% (цель ТЗ ≥70%)")
|
|
for method in ("code", "exact", "embedding", "fuzzy", None):
|
|
print(f" {method or 'unmatched':10}: {by_method[method]} строк")
|
|
|
|
print("\nпримеры сопоставлений:")
|
|
for method in ("code", "exact", "embedding", "fuzzy"):
|
|
for raw, canon, score in samples.get(method, [])[:3]:
|
|
print(f" [{method:9}] «{raw[:36]}» → «{canon[:36]}» ({score})")
|
|
print("\nпримеры unmatched:")
|
|
for raw, _canon, _score in samples.get(None, [])[:6]:
|
|
print(f" «{raw[:52]}»")
|