feat(день2): экран верификации с предложениями + обучение, детектор переплаты

- очередь верификации: предложенный кандидат (топ эмбеддинг), подтвердить/другое/пропустить - POST /match пополняет синонимы (learned_synonym) — обучающаяся нормализация - сравнение: медиана + разброс цен + отклонение клиники (переплата/выгода) - эмбеддер: дедуп названий + ретрай при 429 (квота Gemini) - развёрнуто на med.secondbrain.tools Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
2026-06-27 10:47:57 +05:00
parent 9c0ae2c0bf
commit 1ff5d93cd8
6 changed files with 181 additions and 35 deletions
@@ -8,6 +8,7 @@ from __future__ import annotations

 import json
 import os
+import re
 import sqlite3
 import sys
 from pathlib import Path
@@ -38,6 +39,14 @@ def db() -> sqlite3.Connection:
    return conn


+_WS = re.compile(r"\s+")
+
+
+def _norm(text: str | None) -> str:
+    """Нормализованная форма названия (для запоминания синонима при верификации)."""
+    return _WS.sub(" ", (text or "").strip().lower().replace("ё", "е").replace("-", " ")).strip()
+
+
 def _price_out(row: sqlite3.Row) -> PriceOut:
    return PriceOut(
        partner_id=row["partner_id"],
@@ -136,37 +145,72 @@ def search(q: str = Query(..., min_length=1), limit: int = 20):
    return {"services": services, "partners": partners}


-@app.get("/unmatched", summary="Несопоставленные позиции (для операторов)")
+@app.get("/unmatched", summary="Очередь верификации: позиции с предложенным кандидатом")
 def unmatched(limit: int = 50, offset: int = 0):
-    sql = """SELECT pi.item_id, pi.service_name_raw, pi.prices, p.name AS partner_name
-             FROM price_item pi JOIN partner p ON pi.partner_id = p.partner_id
+    """Несопоставленные позиции с лучшим кандидатом из справочника (сортировка — по уверенности)."""
+    sql = """SELECT pi.item_id, pi.service_name_raw, pi.prices, pi.unit, p.name AS partner_name,
+                    pi.suggested_service_id, pi.suggested_score,
+                    s.name_ru AS suggested_name, s.specialty AS suggested_specialty
+             FROM price_item pi
+             JOIN partner p ON pi.partner_id = p.partner_id
+             LEFT JOIN service s ON pi.suggested_service_id = s.service_id
             WHERE pi.service_id IS NULL AND pi.is_active = 1
-             ORDER BY pi.item_id LIMIT ? OFFSET ?"""
+             ORDER BY pi.suggested_score DESC, pi.item_id
+             LIMIT ? OFFSET ?"""
    with db() as conn:
        return [
            {
                "item_id": r["item_id"],
                "service_name_raw": r["service_name_raw"],
                "partner_name": r["partner_name"],
+                "unit": r["unit"],
                "prices": json.loads(r["prices"] or "{}"),
+                "suggested": (
+                    {
+                        "service_id": r["suggested_service_id"],
+                        "name_ru": r["suggested_name"],
+                        "specialty": r["suggested_specialty"],
+                        "score": r["suggested_score"],
+                    }
+                    if r["suggested_service_id"]
+                    else None
+                ),
            }
            for r in conn.execute(sql, [limit, offset])
        ]


-@app.post("/match", summary="Ручное сопоставление позиции со справочником")
+@app.get("/unmatched/count", summary="Сколько позиций ещё в очереди верификации")
+def unmatched_count():
+    with db() as conn:
+        n = conn.execute(
+            "SELECT COUNT(*) FROM price_item WHERE service_id IS NULL AND is_active = 1"
+        ).fetchone()[0]
+        learned = conn.execute("SELECT COUNT(*) FROM learned_synonym").fetchone()[0]
+    return {"remaining": n, "learned_synonyms": learned}
+
+
+@app.post("/match", summary="Ручное сопоставление позиции (с дообучением)")
 def manual_match(item_id: int, service_id: str):
    with db() as conn:
        if not conn.execute("SELECT 1 FROM service WHERE service_id = ?", [service_id]).fetchone():
            raise HTTPException(404, "услуга справочника не найдена")
-        cur = conn.execute(
+        row = conn.execute(
+            "SELECT service_name_raw FROM price_item WHERE item_id = ?", [item_id]
+        ).fetchone()
+        if not row:
+            raise HTTPException(404, "позиция не найдена")
+        conn.execute(
            "UPDATE price_item SET service_id = ?, map_method = 'manual', map_confidence = 1.0 WHERE item_id = ?",
            [service_id, item_id],
        )
+        # Дообучение: запоминаем синоним — следующие прогоны сопоставят его автоматически.
+        conn.execute(
+            "INSERT OR REPLACE INTO learned_synonym (name_norm, service_id) VALUES (?, ?)",
+            [_norm(row["service_name_raw"]), service_id],
+        )
        conn.commit()
-        if cur.rowcount == 0:
-            raise HTTPException(404, "позиция не найдена")
-    return {"item_id": item_id, "service_id": service_id, "status": "matched"}
+    return {"item_id": item_id, "service_id": service_id, "status": "matched", "learned": True}


@app.get("/stats", response_model=Stats, summary="Сводка качества обработки")
@@ -44,6 +44,9 @@ class MatchResult(BaseModel):
    service_id: str | None = None
    method: str | None = None  # code / exact / embedding / fuzzy / manual
    confidence: float = 0.0
+    # Лучший кандидат для очереди верификации, даже если ниже порога автосопоставления.
+    suggested_service_id: str | None = None
+    suggested_score: float = 0.0


 # --- выдача API ---
@@ -43,6 +43,8 @@ class GeminiEmbedder:
        RETRIEVAL_QUERY: так несвязанные названия расходятся сильнее, чем при
        SEMANTIC_SIMILARITY, и порог отсечения работает надёжнее.
        """
+        import time
+
        from google.genai import types

        items = [t if t and t.strip() else " " for t in texts]
@@ -50,8 +52,16 @@ class GeminiEmbedder:
        vectors: list[list[float]] = []
        for start in range(0, len(items), self.batch):
            chunk = items[start : start + self.batch]
-            response = self._client.models.embed_content(model=self.model, contents=chunk, config=config)
-            vectors.extend(embedding.values for embedding in response.embeddings)
+            for attempt in range(8):
+                try:
+                    response = self._client.models.embed_content(model=self.model, contents=chunk, config=config)
+                    vectors.extend(embedding.values for embedding in response.embeddings)
+                    break
+                except Exception as exc:  # пауза и повтор при превышении квоты (429)
+                    if ("RESOURCE_EXHAUSTED" in str(exc) or "429" in str(exc)) and attempt < 7:
+                        time.sleep(20)
+                        continue
+                    raise

        array = np.array(vectors, dtype=np.float32)
        if normalize_embeddings:
@@ -89,23 +89,35 @@ class Matcher:
            pending_norm.append(name_norm)

        if pending_norm and self.embedder is not None and self.dict_emb is not None:
+            # Дедуп: одинаковые названия эмбеддим один раз (экономит запросы и квоту API).
+            pending_names = [names[i] for i in pending_idx]
+            unique: dict[str, int] = {}
+            for nm in pending_names:
+                unique.setdefault(nm, len(unique))
            query = self.embedder.encode(
-                [names[i] for i in pending_idx],
-                normalize_embeddings=True,
-                task_type="RETRIEVAL_QUERY",
+                list(unique), normalize_embeddings=True, task_type="RETRIEVAL_QUERY"
            )
            sims = query @ self.dict_emb.T  # косинус по нормированным векторам
            best_idx = sims.argmax(axis=1)
            best_score = sims.max(axis=1)
            for k, i in enumerate(pending_idx):
-                if best_score[k] >= self.emb_threshold:
+                u = unique[pending_names[k]]
+                suggested = self.services[int(best_idx[u])].service_id
+                score = float(best_score[u])
+                if score >= self.emb_threshold:
                    results[i] = MatchResult(
-                        service_id=self.services[int(best_idx[k])].service_id,
+                        service_id=suggested,
                        method="embedding",
-                        confidence=float(best_score[k]),
+                        confidence=score,
+                        suggested_service_id=suggested,
+                        suggested_score=score,
                    )
                else:
-                    results[i] = self._match_by_fuzzy(pending_norm[k])
+                    # Не дотянул до порога — в очередь, но кандидата сохраняем оператору.
+                    fallback = self._match_by_fuzzy(pending_norm[k])
+                    fallback.suggested_service_id = suggested
+                    fallback.suggested_score = score
+                    results[i] = fallback
        else:
            for k, i in enumerate(pending_idx):
                results[i] = self._match_by_fuzzy(pending_norm[k])
@@ -37,18 +37,21 @@ CREATE TABLE IF NOT EXISTS price_item (
    item_id INTEGER PRIMARY KEY AUTOINCREMENT, doc_id TEXT, partner_id TEXT,
    service_name_raw TEXT, service_code_source TEXT, service_id TEXT,
    prices TEXT, price_resident REAL, price_nonresident REAL, unit TEXT,
-    effective_date TEXT, map_method TEXT, map_confidence REAL, status TEXT, is_active INTEGER
+    effective_date TEXT, map_method TEXT, map_confidence REAL, status TEXT, is_active INTEGER,
+    suggested_service_id TEXT, suggested_score REAL
 );
+-- Синонимы, выученные при ручной верификации (для дообучения нормализации).
+CREATE TABLE IF NOT EXISTS learned_synonym (name_norm TEXT PRIMARY KEY, service_id TEXT);
 CREATE INDEX IF NOT EXISTS price_item_service ON price_item(service_id);
 CREATE INDEX IF NOT EXISTS price_item_partner ON price_item(partner_id);
 """


 def _open(db_path: str) -> sqlite3.Connection:
+    # Полная пересборка: удаляем файл, чтобы схема всегда создавалась свежей.
+    Path(db_path).unlink(missing_ok=True)
    conn = sqlite3.connect(db_path)
    conn.executescript(SCHEMA)
-    for table in ("price_item", "price_document", "partner", "service"):
-        conn.execute(f"DELETE FROM {table}")  # пересборка с нуля при каждом прогоне
    return conn


@@ -102,11 +105,12 @@ def run(data_dir: str, db_path: str, dict_path: str, embedder=None, use_vision:
            """INSERT INTO price_item
               (doc_id, partner_id, service_name_raw, service_code_source, service_id, prices,
                price_resident, price_nonresident, unit, effective_date, map_method, map_confidence,
-                status, is_active)
-               VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,1)""",
+                status, is_active, suggested_service_id, suggested_score)
+               VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,1,?,?)""",
            (doc_id, partner_id, raw, code, match.service_id, json.dumps(prices, ensure_ascii=False),
             prices.get("resident"), prices.get("nonresident"), unit, eff_iso, match.method,
-             round(match.confidence, 3), status),
+             round(match.confidence, 3), status, match.suggested_service_id,
+             round(match.suggested_score, 3) if match.suggested_score else None),
        )
        report["items"] += 1

@@ -46,12 +46,17 @@
    <div id="compare" class="mt-6"></div>
  </section>

-  <!-- Админка -->
+  <!-- Админка + верификация -->
  <section id="view-admin" class="hidden">
    <h2 class="text-2xl font-bold mt-6">Дашборд обработки</h2>
    <div id="stats" class="grid grid-cols-2 sm:grid-cols-4 gap-3 mt-4"></div>
-    <h3 class="text-lg font-semibold mt-8">Очередь ручной разметки (unmatched)</h3>
-    <div id="unmatched" class="mt-3 bg-white border border-[#ebedf0] rounded-xl overflow-hidden"></div>
+
+    <div class="flex items-center justify-between mt-8">
+      <h3 class="text-lg font-semibold">Очередь верификации</h3>
+      <div id="vprogress" class="text-sm text-tg"></div>
+    </div>
+    <p class="text-sm text-tg mt-1">Система предлагает кандидата из справочника — подтвердите, выберите другой или пропустите. Подтверждение обучает систему.</p>
+    <div id="queue" class="mt-3 space-y-2"></div>
  </section>

 </main>
@@ -88,26 +93,37 @@ async function search() {
 async function compare(serviceId, name) {
  $('suggest').classList.add('hidden');
  const rows = await fetch(`${API}/services/${encodeURIComponent(serviceId)}/partners`).then(r => r.json());
-  const best = rows.reduce((m, r) => (r.price_resident_kzt != null && (m == null || r.price_resident_kzt < m)) ? r.price_resident_kzt : m, null);
+  const vals = rows.map(r => r.price_resident_kzt).filter(v => v != null).sort((a, b) => a - b);
+  const best = vals.length ? vals[0] : null;
+  const worst = vals.length ? vals[vals.length - 1] : null;
+  const median = vals.length ? (vals.length % 2 ? vals[(vals.length - 1) / 2] : (vals[vals.length / 2 - 1] + vals[vals.length / 2]) / 2) : null;
+  const dev = (v) => (median && v != null) ? Math.round((v - median) / median * 100) : null;
+  const spread = (best != null && worst != null) ? worst - best : null;
  $('compare').innerHTML = `
    <div class="bg-white border border-[#ebedf0] rounded-2xl shadow-sm overflow-hidden">
      <div class="px-5 py-4 border-b border-[#ebedf0]">
        <div class="text-lg font-semibold">${name}</div>
-        <div class="text-sm text-tg">${rows.length} клиник(и) оказывают услугу</div>
+        <div class="text-sm text-tg">${rows.length} клиник(и) · медиана ${money(median)}${spread ? ' · разброс цен до <b class="text-nomad">' + money(spread) + '</b>' : ''}</div>
      </div>
      <table class="w-full text-sm">
        <thead class="bg-lg text-tg text-left">
          <tr><th class="px-5 py-2.5 font-semibold">Клиника</th>
              <th class="px-5 py-2.5 font-semibold text-right">Резидент</th>
+              <th class="px-5 py-2.5 font-semibold text-right">vs медиана</th>
              <th class="px-5 py-2.5 font-semibold text-right">Нерезидент</th>
              <th class="px-5 py-2.5 font-semibold text-right">Прайс от</th></tr>
        </thead>
        <tbody class="divide-y divide-[#f0f0f0]">
          ${rows.map(r => {
            const cheapest = r.price_resident_kzt != null && r.price_resident_kzt === best;
+            const d = dev(r.price_resident_kzt);
+            const badge = d == null ? '—' : (d > 0
+              ? `<span class="text-[#dc2626] font-medium">+${d}%</span>`
+              : `<span class="text-[#0f973d] font-medium">${d}%</span>`);
            return `<tr class="${cheapest ? 'bg-nomad-soft' : ''}">
              <td class="px-5 py-3">${r.partner_name}${cheapest ? ' <span class="text-nomad text-xs font-semibold">· выгодно</span>' : ''}</td>
              <td class="px-5 py-3 text-right tabular-nums font-medium">${money(r.price_resident_kzt)}</td>
+              <td class="px-5 py-3 text-right tabular-nums text-xs">${badge}</td>
              <td class="px-5 py-3 text-right tabular-nums text-tg">${money(r.price_nonresident_kzt)}</td>
              <td class="px-5 py-3 text-right text-tg text-xs">${r.effective_date || '—'}</td></tr>`;
          }).join('')}
@@ -125,12 +141,69 @@ async function loadAdmin() {
    <div class="bg-white border border-[#ebedf0] rounded-xl p-4">
      <div class="text-2xl font-bold text-nomad">${v}</div>
      <div class="text-xs text-tg mt-1">${l}</div></div>`).join('');
-  const u = await fetch(`${API}/unmatched?limit=25`).then(r => r.json());
-  $('unmatched').innerHTML = `<table class="w-full text-sm"><tbody class="divide-y divide-[#f0f0f0]">
-    ${u.map(x => `<tr><td class="px-5 py-2.5">${x.service_name_raw}</td>
-      <td class="px-5 py-2.5 text-tg text-xs">${x.partner_name}</td>
-      <td class="px-5 py-2.5 text-right tabular-nums text-tg">${Object.values(x.prices)[0] != null ? money(Object.values(x.prices)[0]) : '—'}</td></tr>`).join('')}
-    </tbody></table>`;
+  loadQueue();
+}
+
+async function loadQueue() {
+  const items = await fetch(`${API}/unmatched?limit=40`).then(r => r.json());
+  $('queue').innerHTML = items.map(renderCard).join('') || '<div class="text-tg p-4">Очередь пуста 🎉</div>';
+  loadCounts();
+}
+
+async function loadCounts() {
+  const c = await fetch(`${API}/unmatched/count`).then(r => r.json());
+  $('vprogress').innerHTML = `осталось <b>${c.remaining}</b> · обучено синонимов <b class="text-nomad">${c.learned_synonyms}</b>`;
+}
+
+function renderCard(it) {
+  const price = Object.values(it.prices)[0];
+  const sug = it.suggested;
+  const sugHtml = sug
+    ? `<div class="text-sm mt-1">предложение: <b>${sug.name_ru}</b> <span class="text-tg">${sug.specialty || ''} · ${(sug.score || 0).toFixed(2)}</span></div>`
+    : `<div class="text-sm text-tg mt-1">кандидата нет</div>`;
+  const confirm = sug
+    ? `<button onclick="doMatch(${it.item_id}, '${sug.service_id}', this)" class="bg-nomad text-white text-sm rounded-lg px-3 py-1.5 hover:bg-nomad-dark">✓ Подтвердить</button>`
+    : '';
+  return `<div id="card-${it.item_id}" class="bg-white border border-[#ebedf0] rounded-xl p-4">
+    <div class="flex justify-between items-start gap-3">
+      <div>
+        <div>извлечено: <b>${it.service_name_raw}</b></div>
+        <div class="text-sm text-tg">${it.partner_name}${price != null ? ' · ' + money(price) : ''}</div>
+        ${sugHtml}
+      </div>
+      <div class="flex gap-2 shrink-0">
+        ${confirm}
+        <button onclick="togglePick(${it.item_id})" class="border border-[#e5e7eb] text-sm rounded-lg px-3 py-1.5 hover:bg-lg">Другое</button>
+        <button onclick="skipCard(${it.item_id})" class="text-tg text-sm rounded-lg px-3 py-1.5 hover:bg-lg">Пропустить</button>
+      </div>
+    </div>
+    <div id="pick-${it.item_id}" class="hidden mt-3">
+      <input oninput="searchDict(${it.item_id}, this.value)" placeholder="найти услугу в справочнике…"
+        class="w-full bg-white border border-[#d1d5db] rounded-lg px-3 py-2 text-sm outline-none focus:border-nomad">
+      <div id="dict-${it.item_id}" class="mt-1 bg-white border border-[#ebedf0] rounded-lg divide-y divide-[#f0f0f0] max-h-44 overflow-auto"></div>
+    </div>
+  </div>`;
+}
+
+async function doMatch(itemId, serviceId, btn) {
+  if (btn) btn.disabled = true;
+  await fetch(`${API}/match?item_id=${itemId}&service_id=${encodeURIComponent(serviceId)}`, { method: 'POST' });
+  const card = $(`card-${itemId}`); if (card) card.remove();
+  loadCounts();
+}
+
+function togglePick(id) { $(`pick-${id}`).classList.toggle('hidden'); }
+function skipCard(id) { const c = $(`card-${id}`); if (c) c.remove(); }
+
+let dictTimer;
+function searchDict(itemId, q) {
+  clearTimeout(dictTimer);
+  dictTimer = setTimeout(async () => {
+    if (q.trim().length < 2) return;
+    const items = await fetch(`${API}/services?q=${encodeURIComponent(q)}&limit=8`).then(r => r.json());
+    $(`dict-${itemId}`).innerHTML = items.map(s =>
+      `<button class="block w-full text-left px-3 py-2 text-sm hover:bg-nomad-soft" onclick="doMatch(${itemId}, '${s.service_id}', null)">${s.name_ru} <span class="text-tg text-xs">${s.specialty || ''}</span></button>`).join('');
+  }, 250);
 }
 </script>
 </body>