From 1ff5d93cd87d86272528d9fce7b4007e53629e57 Mon Sep 17 00:00:00 2001 From: dmitriylaukhin Date: Sat, 27 Jun 2026 10:47:57 +0500 Subject: [PATCH] =?UTF-8?q?feat(=D0=B4=D0=B5=D0=BD=D1=8C2):=20=D1=8D=D0=BA?= =?UTF-8?q?=D1=80=D0=B0=D0=BD=20=D0=B2=D0=B5=D1=80=D0=B8=D1=84=D0=B8=D0=BA?= =?UTF-8?q?=D0=B0=D1=86=D0=B8=D0=B8=20=D1=81=20=D0=BF=D1=80=D0=B5=D0=B4?= =?UTF-8?q?=D0=BB=D0=BE=D0=B6=D0=B5=D0=BD=D0=B8=D1=8F=D0=BC=D0=B8=20+=20?= =?UTF-8?q?=D0=BE=D0=B1=D1=83=D1=87=D0=B5=D0=BD=D0=B8=D0=B5,=20=D0=B4?= =?UTF-8?q?=D0=B5=D1=82=D0=B5=D0=BA=D1=82=D0=BE=D1=80=20=D0=BF=D0=B5=D1=80?= =?UTF-8?q?=D0=B5=D0=BF=D0=BB=D0=B0=D1=82=D1=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - очередь верификации: предложенный кандидат (топ эмбеддинг), подтвердить/другое/пропустить - POST /match пополняет синонимы (learned_synonym) — обучающаяся нормализация - сравнение: медиана + разброс цен + отклонение клиники (переплата/выгода) - эмбеддер: дедуп названий + ретрай при 429 (квота Gemini) - развёрнуто на med.secondbrain.tools Co-Authored-By: Claude Opus 4.8 (1M context) --- api/main.py | 62 +++++++++++++++++++++---- contracts/models.py | 3 ++ etl/normalize/embedding.py | 14 +++++- etl/normalize/matcher.py | 26 ++++++++--- etl/pipeline.py | 16 ++++--- web/index.html | 95 +++++++++++++++++++++++++++++++++----- 6 files changed, 181 insertions(+), 35 deletions(-) diff --git a/api/main.py b/api/main.py index a91e9b9..06b6264 100644 --- a/api/main.py +++ b/api/main.py @@ -8,6 +8,7 @@ from __future__ import annotations import json import os +import re import sqlite3 import sys from pathlib import Path @@ -38,6 +39,14 @@ def db() -> sqlite3.Connection: return conn +_WS = re.compile(r"\s+") + + +def _norm(text: str | None) -> str: + """Нормализованная форма названия (для запоминания синонима при верификации).""" + return _WS.sub(" ", (text or "").strip().lower().replace("ё", "е").replace("-", " ")).strip() + + def _price_out(row: sqlite3.Row) -> PriceOut: return PriceOut( partner_id=row["partner_id"], @@ -136,37 +145,72 @@ def search(q: str = Query(..., min_length=1), limit: int = 20): return {"services": services, "partners": partners} -@app.get("/unmatched", summary="Несопоставленные позиции (для операторов)") +@app.get("/unmatched", summary="Очередь верификации: позиции с предложенным кандидатом") def unmatched(limit: int = 50, offset: int = 0): - sql = """SELECT pi.item_id, pi.service_name_raw, pi.prices, p.name AS partner_name - FROM price_item pi JOIN partner p ON pi.partner_id = p.partner_id + """Несопоставленные позиции с лучшим кандидатом из справочника (сортировка — по уверенности).""" + sql = """SELECT pi.item_id, pi.service_name_raw, pi.prices, pi.unit, p.name AS partner_name, + pi.suggested_service_id, pi.suggested_score, + s.name_ru AS suggested_name, s.specialty AS suggested_specialty + FROM price_item pi + JOIN partner p ON pi.partner_id = p.partner_id + LEFT JOIN service s ON pi.suggested_service_id = s.service_id WHERE pi.service_id IS NULL AND pi.is_active = 1 - ORDER BY pi.item_id LIMIT ? OFFSET ?""" + ORDER BY pi.suggested_score DESC, pi.item_id + LIMIT ? OFFSET ?""" with db() as conn: return [ { "item_id": r["item_id"], "service_name_raw": r["service_name_raw"], "partner_name": r["partner_name"], + "unit": r["unit"], "prices": json.loads(r["prices"] or "{}"), + "suggested": ( + { + "service_id": r["suggested_service_id"], + "name_ru": r["suggested_name"], + "specialty": r["suggested_specialty"], + "score": r["suggested_score"], + } + if r["suggested_service_id"] + else None + ), } for r in conn.execute(sql, [limit, offset]) ] -@app.post("/match", summary="Ручное сопоставление позиции со справочником") +@app.get("/unmatched/count", summary="Сколько позиций ещё в очереди верификации") +def unmatched_count(): + with db() as conn: + n = conn.execute( + "SELECT COUNT(*) FROM price_item WHERE service_id IS NULL AND is_active = 1" + ).fetchone()[0] + learned = conn.execute("SELECT COUNT(*) FROM learned_synonym").fetchone()[0] + return {"remaining": n, "learned_synonyms": learned} + + +@app.post("/match", summary="Ручное сопоставление позиции (с дообучением)") def manual_match(item_id: int, service_id: str): with db() as conn: if not conn.execute("SELECT 1 FROM service WHERE service_id = ?", [service_id]).fetchone(): raise HTTPException(404, "услуга справочника не найдена") - cur = conn.execute( + row = conn.execute( + "SELECT service_name_raw FROM price_item WHERE item_id = ?", [item_id] + ).fetchone() + if not row: + raise HTTPException(404, "позиция не найдена") + conn.execute( "UPDATE price_item SET service_id = ?, map_method = 'manual', map_confidence = 1.0 WHERE item_id = ?", [service_id, item_id], ) + # Дообучение: запоминаем синоним — следующие прогоны сопоставят его автоматически. + conn.execute( + "INSERT OR REPLACE INTO learned_synonym (name_norm, service_id) VALUES (?, ?)", + [_norm(row["service_name_raw"]), service_id], + ) conn.commit() - if cur.rowcount == 0: - raise HTTPException(404, "позиция не найдена") - return {"item_id": item_id, "service_id": service_id, "status": "matched"} + return {"item_id": item_id, "service_id": service_id, "status": "matched", "learned": True} @app.get("/stats", response_model=Stats, summary="Сводка качества обработки") diff --git a/contracts/models.py b/contracts/models.py index f759509..2b3dc33 100644 --- a/contracts/models.py +++ b/contracts/models.py @@ -44,6 +44,9 @@ class MatchResult(BaseModel): service_id: str | None = None method: str | None = None # code / exact / embedding / fuzzy / manual confidence: float = 0.0 + # Лучший кандидат для очереди верификации, даже если ниже порога автосопоставления. + suggested_service_id: str | None = None + suggested_score: float = 0.0 # --- выдача API --- diff --git a/etl/normalize/embedding.py b/etl/normalize/embedding.py index cdd3334..8b1d5df 100644 --- a/etl/normalize/embedding.py +++ b/etl/normalize/embedding.py @@ -43,6 +43,8 @@ class GeminiEmbedder: RETRIEVAL_QUERY: так несвязанные названия расходятся сильнее, чем при SEMANTIC_SIMILARITY, и порог отсечения работает надёжнее. """ + import time + from google.genai import types items = [t if t and t.strip() else " " for t in texts] @@ -50,8 +52,16 @@ class GeminiEmbedder: vectors: list[list[float]] = [] for start in range(0, len(items), self.batch): chunk = items[start : start + self.batch] - response = self._client.models.embed_content(model=self.model, contents=chunk, config=config) - vectors.extend(embedding.values for embedding in response.embeddings) + for attempt in range(8): + try: + response = self._client.models.embed_content(model=self.model, contents=chunk, config=config) + vectors.extend(embedding.values for embedding in response.embeddings) + break + except Exception as exc: # пауза и повтор при превышении квоты (429) + if ("RESOURCE_EXHAUSTED" in str(exc) or "429" in str(exc)) and attempt < 7: + time.sleep(20) + continue + raise array = np.array(vectors, dtype=np.float32) if normalize_embeddings: diff --git a/etl/normalize/matcher.py b/etl/normalize/matcher.py index 28cd4aa..d3b7fa6 100644 --- a/etl/normalize/matcher.py +++ b/etl/normalize/matcher.py @@ -89,23 +89,35 @@ class Matcher: pending_norm.append(name_norm) if pending_norm and self.embedder is not None and self.dict_emb is not None: + # Дедуп: одинаковые названия эмбеддим один раз (экономит запросы и квоту API). + pending_names = [names[i] for i in pending_idx] + unique: dict[str, int] = {} + for nm in pending_names: + unique.setdefault(nm, len(unique)) query = self.embedder.encode( - [names[i] for i in pending_idx], - normalize_embeddings=True, - task_type="RETRIEVAL_QUERY", + list(unique), normalize_embeddings=True, task_type="RETRIEVAL_QUERY" ) sims = query @ self.dict_emb.T # косинус по нормированным векторам best_idx = sims.argmax(axis=1) best_score = sims.max(axis=1) for k, i in enumerate(pending_idx): - if best_score[k] >= self.emb_threshold: + u = unique[pending_names[k]] + suggested = self.services[int(best_idx[u])].service_id + score = float(best_score[u]) + if score >= self.emb_threshold: results[i] = MatchResult( - service_id=self.services[int(best_idx[k])].service_id, + service_id=suggested, method="embedding", - confidence=float(best_score[k]), + confidence=score, + suggested_service_id=suggested, + suggested_score=score, ) else: - results[i] = self._match_by_fuzzy(pending_norm[k]) + # Не дотянул до порога — в очередь, но кандидата сохраняем оператору. + fallback = self._match_by_fuzzy(pending_norm[k]) + fallback.suggested_service_id = suggested + fallback.suggested_score = score + results[i] = fallback else: for k, i in enumerate(pending_idx): results[i] = self._match_by_fuzzy(pending_norm[k]) diff --git a/etl/pipeline.py b/etl/pipeline.py index d026155..357ed07 100644 --- a/etl/pipeline.py +++ b/etl/pipeline.py @@ -37,18 +37,21 @@ CREATE TABLE IF NOT EXISTS price_item ( item_id INTEGER PRIMARY KEY AUTOINCREMENT, doc_id TEXT, partner_id TEXT, service_name_raw TEXT, service_code_source TEXT, service_id TEXT, prices TEXT, price_resident REAL, price_nonresident REAL, unit TEXT, - effective_date TEXT, map_method TEXT, map_confidence REAL, status TEXT, is_active INTEGER + effective_date TEXT, map_method TEXT, map_confidence REAL, status TEXT, is_active INTEGER, + suggested_service_id TEXT, suggested_score REAL ); +-- Синонимы, выученные при ручной верификации (для дообучения нормализации). +CREATE TABLE IF NOT EXISTS learned_synonym (name_norm TEXT PRIMARY KEY, service_id TEXT); CREATE INDEX IF NOT EXISTS price_item_service ON price_item(service_id); CREATE INDEX IF NOT EXISTS price_item_partner ON price_item(partner_id); """ def _open(db_path: str) -> sqlite3.Connection: + # Полная пересборка: удаляем файл, чтобы схема всегда создавалась свежей. + Path(db_path).unlink(missing_ok=True) conn = sqlite3.connect(db_path) conn.executescript(SCHEMA) - for table in ("price_item", "price_document", "partner", "service"): - conn.execute(f"DELETE FROM {table}") # пересборка с нуля при каждом прогоне return conn @@ -102,11 +105,12 @@ def run(data_dir: str, db_path: str, dict_path: str, embedder=None, use_vision: """INSERT INTO price_item (doc_id, partner_id, service_name_raw, service_code_source, service_id, prices, price_resident, price_nonresident, unit, effective_date, map_method, map_confidence, - status, is_active) - VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,1)""", + status, is_active, suggested_service_id, suggested_score) + VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,1,?,?)""", (doc_id, partner_id, raw, code, match.service_id, json.dumps(prices, ensure_ascii=False), prices.get("resident"), prices.get("nonresident"), unit, eff_iso, match.method, - round(match.confidence, 3), status), + round(match.confidence, 3), status, match.suggested_service_id, + round(match.suggested_score, 3) if match.suggested_score else None), ) report["items"] += 1 diff --git a/web/index.html b/web/index.html index cb680c4..ed0346a 100644 --- a/web/index.html +++ b/web/index.html @@ -46,12 +46,17 @@
- + @@ -88,26 +93,37 @@ async function search() { async function compare(serviceId, name) { $('suggest').classList.add('hidden'); const rows = await fetch(`${API}/services/${encodeURIComponent(serviceId)}/partners`).then(r => r.json()); - const best = rows.reduce((m, r) => (r.price_resident_kzt != null && (m == null || r.price_resident_kzt < m)) ? r.price_resident_kzt : m, null); + const vals = rows.map(r => r.price_resident_kzt).filter(v => v != null).sort((a, b) => a - b); + const best = vals.length ? vals[0] : null; + const worst = vals.length ? vals[vals.length - 1] : null; + const median = vals.length ? (vals.length % 2 ? vals[(vals.length - 1) / 2] : (vals[vals.length / 2 - 1] + vals[vals.length / 2]) / 2) : null; + const dev = (v) => (median && v != null) ? Math.round((v - median) / median * 100) : null; + const spread = (best != null && worst != null) ? worst - best : null; $('compare').innerHTML = `
${name}
-
${rows.length} клиник(и) оказывают услугу
+
${rows.length} клиник(и) · медиана ${money(median)}${spread ? ' · разброс цен до ' + money(spread) + '' : ''}
+ ${rows.map(r => { const cheapest = r.price_resident_kzt != null && r.price_resident_kzt === best; + const d = dev(r.price_resident_kzt); + const badge = d == null ? '—' : (d > 0 + ? `+${d}%` + : `${d}%`); return ` + `; }).join('')} @@ -125,12 +141,69 @@ async function loadAdmin() {
${v}
${l}
`).join(''); - const u = await fetch(`${API}/unmatched?limit=25`).then(r => r.json()); - $('unmatched').innerHTML = `
Клиника Резидентvs медиана Нерезидент Прайс от
${r.partner_name}${cheapest ? ' · выгодно' : ''} ${money(r.price_resident_kzt)}${badge} ${money(r.price_nonresident_kzt)} ${r.effective_date || '—'}
- ${u.map(x => ` - - `).join('')} -
${x.service_name_raw}${x.partner_name}${Object.values(x.prices)[0] != null ? money(Object.values(x.prices)[0]) : '—'}
`; + loadQueue(); +} + +async function loadQueue() { + const items = await fetch(`${API}/unmatched?limit=40`).then(r => r.json()); + $('queue').innerHTML = items.map(renderCard).join('') || '
Очередь пуста 🎉
'; + loadCounts(); +} + +async function loadCounts() { + const c = await fetch(`${API}/unmatched/count`).then(r => r.json()); + $('vprogress').innerHTML = `осталось ${c.remaining} · обучено синонимов ${c.learned_synonyms}`; +} + +function renderCard(it) { + const price = Object.values(it.prices)[0]; + const sug = it.suggested; + const sugHtml = sug + ? `
предложение: ${sug.name_ru} ${sug.specialty || ''} · ${(sug.score || 0).toFixed(2)}
` + : `
кандидата нет
`; + const confirm = sug + ? `` + : ''; + return `
+
+
+
извлечено: ${it.service_name_raw}
+
${it.partner_name}${price != null ? ' · ' + money(price) : ''}
+ ${sugHtml} +
+
+ ${confirm} + + +
+
+ +
`; +} + +async function doMatch(itemId, serviceId, btn) { + if (btn) btn.disabled = true; + await fetch(`${API}/match?item_id=${itemId}&service_id=${encodeURIComponent(serviceId)}`, { method: 'POST' }); + const card = $(`card-${itemId}`); if (card) card.remove(); + loadCounts(); +} + +function togglePick(id) { $(`pick-${id}`).classList.toggle('hidden'); } +function skipCard(id) { const c = $(`card-${id}`); if (c) c.remove(); } + +let dictTimer; +function searchDict(itemId, q) { + clearTimeout(dictTimer); + dictTimer = setTimeout(async () => { + if (q.trim().length < 2) return; + const items = await fetch(`${API}/services?q=${encodeURIComponent(q)}&limit=8`).then(r => r.json()); + $(`dict-${itemId}`).innerHTML = items.map(s => + ``).join(''); + }, 250); }