admins
1de66eaa85
- эмбеддинги переведены с локального torch на Gemini API (лёгкий контейнер) GeminiEmbedder: gemini-embedding-001, 768d, RETRIEVAL query/document - фильтр настоящих названий: коды/числа не уходят в сопоставление - порог косинуса 0.70 → ~73% автонормализации (цель ТЗ ≥70%), остальное в unmatched - Vision (gemini-2.5-flash, новый SDK) подключён в диспетчер для скан/нулевых PDF; проверено: Клиника 5 — 34 чистые позиции со страницы - зависимости: убран torch/sentence-transformers, добавлен google-genai+numpy Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
80 lines
3.1 KiB
Python
80 lines
3.1 KiB
Python
"""Подбор порога нормализации: выборка позиций → каскад → покрытие при порогах."""
|
|
import random
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
import numpy as np
|
|
|
|
REPO = Path(__file__).resolve().parents[2]
|
|
sys.path.insert(0, str(REPO))
|
|
|
|
from rapidfuzz import fuzz, process
|
|
|
|
from etl.dictionary import load_dictionary, normalize_name
|
|
from etl.extractors import extract
|
|
from etl.normalize.embedding import GeminiEmbedder
|
|
from etl.normalize.matcher import _CODE_CORE_RE
|
|
|
|
rows = []
|
|
for path in sorted((REPO / "data/raw").glob("*")):
|
|
for row in extract(str(path)).rows:
|
|
rows.append((row.service_name_raw, row.service_code_source))
|
|
random.seed(0)
|
|
sample = random.sample(rows, min(2000, len(rows)))
|
|
print(f"всего {len(rows)}, выборка {len(sample)}")
|
|
|
|
services = load_dictionary(REPO / "data/reference/dictionary.xlsx")
|
|
by_code = {s.tarificator_code: s for s in services if s.tarificator_code}
|
|
by_norm: dict[str, object] = {}
|
|
for s in services:
|
|
by_norm.setdefault(s.name_norm, s)
|
|
names_norm = [s.name_norm for s in services]
|
|
|
|
embedder = GeminiEmbedder()
|
|
print("эмбеддинг справочника (1281)…")
|
|
dict_emb = embedder.encode([s.name_ru for s in services], task_type="RETRIEVAL_DOCUMENT")
|
|
|
|
code_n = exact_n = 0
|
|
pending: list[tuple[str, str]] = []
|
|
for name, code in sample:
|
|
m = _CODE_CORE_RE.search(code) if code else None
|
|
if m and m.group(0) in by_code:
|
|
code_n += 1
|
|
continue
|
|
norm = normalize_name(name)
|
|
if norm in by_norm:
|
|
exact_n += 1
|
|
continue
|
|
pending.append((name, norm))
|
|
|
|
print(f"код: {code_n} | точное: {exact_n} | дальше эмбеддинги/fuzzy: {len(pending)}")
|
|
print("эмбеддинг запросов…")
|
|
query_emb = embedder.encode([p[0] for p in pending], task_type="RETRIEVAL_QUERY")
|
|
sims = query_emb @ dict_emb.T
|
|
top_idx = sims.argmax(axis=1)
|
|
top_score = sims.max(axis=1)
|
|
fuzzy_score = np.array(
|
|
[process.extractOne(nn, names_norm, scorer=fuzz.token_set_ratio)[1] for _, nn in pending]
|
|
)
|
|
|
|
base = code_n + exact_n
|
|
print(f"\nБаза (код+точное): {base}/{len(sample)} = {100 * base / len(sample):.0f}%")
|
|
for thr in (0.58, 0.60, 0.62, 0.64, 0.66, 0.70):
|
|
emb_matched = int((top_score >= thr).sum())
|
|
fuzzy_matched = int(((fuzzy_score >= 88) & (top_score < thr)).sum())
|
|
coverage = base + emb_matched + fuzzy_matched
|
|
print(f" порог {thr}: эмб {emb_matched} + fuzzy {fuzzy_matched} → покрытие {100 * coverage / len(sample):.0f}%")
|
|
|
|
print("\nпримеры эмбеддинг-совпадений (порог 0.62):")
|
|
shown = 0
|
|
for k, (name, _) in enumerate(pending):
|
|
if top_score[k] >= 0.62 and shown < 7:
|
|
print(f" «{name[:34]}» → «{services[int(top_idx[k])].name_ru[:34]}» ({top_score[k]:.2f})")
|
|
shown += 1
|
|
print("примеры unmatched (top<0.62 и fuzzy<88):")
|
|
shown = 0
|
|
for k, (name, _) in enumerate(pending):
|
|
if top_score[k] < 0.62 and fuzzy_score[k] < 88 and shown < 6:
|
|
print(f" «{name[:48]}» (лучший {services[int(top_idx[k])].name_ru[:22]} {top_score[k]:.2f})")
|
|
shown += 1
|