feat: конвейер ingest→SQLite, API на БД, фронт сравнения цен

- etl/pipeline.py — extract→normalize→validate→SQLite, дедуп партнёров, версии цен, отчёт качества
- api/main.py — 7 эндпоинтов на SQLite; ядро /services/{id}/partners (сравнение)
- web/index.html — одностраничный фронт (палитра Nomad): поиск, сравнение, дашборд, unmatched
- загрузчик справочника: уникальный service_id

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
2026-06-26 17:30:10 +05:00
parent 1de66eaa85
commit 985faadf36
6 changed files with 438 additions and 24 deletions
+121 -20
View File
@@ -1,69 +1,170 @@
"""MedArchive API — эндпоинты ТЗ §4.5 плюс /stats для дашборда.
Здесь зафиксирован контракт API (пути, параметры, модели ответа). Реализация
запросов к БД — задача агента D; пока эндпоинты возвращают пустые заготовки,
чтобы приложение поднималось и фронт мог разрабатываться на сгенерированной
OpenAPI-схеме.
Читает базу, собранную конвейером (etl/pipeline.py). Для MVP это SQLite; путь
задаётся переменной окружения MEDARCHIVE_DB. Ядро WOW — /services/{id}/partners:
«кто оказывает услугу и по какой цене», отсортировано от выгодной.
"""
from __future__ import annotations
import json
import os
import sqlite3
import sys
from pathlib import Path
from fastapi import FastAPI, Query
from fastapi import FastAPI, HTTPException, Query
from fastapi.middleware.cors import CORSMiddleware
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
from contracts.models import PartnerOut, PriceOut, ServiceOut, Stats # noqa: E402
DB_PATH = os.environ.get("MEDARCHIVE_DB", str(Path(__file__).resolve().parents[1] / "data/medarchive.db"))
app = FastAPI(
title="MedArchive API",
version="0.1.0",
version="1.0.0",
description="Поиск услуг и цен по архиву прайсов клиник-партнёров.",
)
app.add_middleware(
CORSMiddleware, allow_origins=["*"], allow_methods=["*"], allow_headers=["*"]
)
def db() -> sqlite3.Connection:
conn = sqlite3.connect(DB_PATH)
conn.row_factory = sqlite3.Row
return conn
def _price_out(row: sqlite3.Row) -> PriceOut:
return PriceOut(
partner_id=row["partner_id"],
partner_name=row["partner_name"],
price_resident_kzt=row["price_resident"],
price_nonresident_kzt=row["price_nonresident"],
prices=json.loads(row["prices"] or "{}"),
effective_date=row["effective_date"],
)
@app.get("/services", response_model=list[ServiceOut], summary="Услуги справочника")
def list_services(specialty: str | None = Query(None, description="Фильтр по специальности")):
return [] # TODO(D): выборка из таблицы service
def list_services(specialty: str | None = None, q: str | None = None, limit: int = 100):
sql = "SELECT service_id, specialty, name_ru, tarificator_code FROM service WHERE 1=1"
args: list = []
if specialty:
sql += " AND specialty = ?"
args.append(specialty)
if q:
sql += " AND name_ru LIKE ?"
args.append(f"%{q}%")
sql += " ORDER BY name_ru LIMIT ?"
args.append(limit)
with db() as conn:
return [ServiceOut(**dict(r)) for r in conn.execute(sql, args)]
@app.get("/services/{service_id}/partners", response_model=list[PriceOut],
summary="Кто оказывает услугу и по какой цене")
def service_partners(service_id: str):
return [] # TODO(D): партнёры с ценами по услуге (ядро WOW-сравнения)
sql = """SELECT pi.partner_id, p.name AS partner_name, pi.price_resident, pi.price_nonresident,
pi.prices, pi.effective_date
FROM price_item pi JOIN partner p ON pi.partner_id = p.partner_id
WHERE pi.service_id = ? AND pi.is_active = 1
ORDER BY COALESCE(pi.price_resident, pi.price_nonresident)"""
with db() as conn:
return [_price_out(r) for r in conn.execute(sql, [service_id])]
@app.get("/partners", response_model=list[PartnerOut], summary="Партнёры")
def list_partners(city: str | None = None, is_active: bool | None = None):
return [] # TODO(D)
def list_partners(city: str | None = None):
sql = "SELECT partner_id, name, city FROM partner WHERE 1=1"
args: list = []
if city:
sql += " AND city = ?"
args.append(city)
sql += " ORDER BY name"
with db() as conn:
return [PartnerOut(partner_id=r["partner_id"], name=r["name"], city=r["city"]) for r in conn.execute(sql, args)]
@app.get("/partners/{partner_id}/services", response_model=list[PriceOut],
summary="Все услуги партнёра с ценами")
def partner_services(partner_id: str):
return [] # TODO(D)
def partner_services(partner_id: str, limit: int = 500):
sql = """SELECT pi.partner_id, p.name AS partner_name, pi.price_resident, pi.price_nonresident,
pi.prices, pi.effective_date
FROM price_item pi JOIN partner p ON pi.partner_id = p.partner_id
WHERE pi.partner_id = ? AND pi.is_active = 1 LIMIT ?"""
with db() as conn:
return [_price_out(r) for r in conn.execute(sql, [partner_id, limit])]
@app.get("/search", summary="Полнотекстовый поиск по услугам и партнёрам")
def search(q: str = Query(..., min_length=1)):
return {"services": [], "partners": []} # TODO(D)
def search(q: str = Query(..., min_length=1), limit: int = 20):
with db() as conn:
services = [
dict(r)
for r in conn.execute(
"SELECT service_id, specialty, name_ru FROM service WHERE name_ru LIKE ? LIMIT ?",
[f"%{q}%", limit],
)
]
partners = [
dict(r)
for r in conn.execute(
"SELECT partner_id, name, city FROM partner WHERE name LIKE ? LIMIT ?", [f"%{q}%", limit]
)
]
return {"services": services, "partners": partners}
@app.get("/unmatched", summary="Несопоставленные позиции (для операторов)")
def unmatched(limit: int = 50, offset: int = 0):
return [] # TODO(D)
sql = """SELECT pi.item_id, pi.service_name_raw, pi.prices, p.name AS partner_name
FROM price_item pi JOIN partner p ON pi.partner_id = p.partner_id
WHERE pi.service_id IS NULL AND pi.is_active = 1
ORDER BY pi.item_id LIMIT ? OFFSET ?"""
with db() as conn:
return [
{
"item_id": r["item_id"],
"service_name_raw": r["service_name_raw"],
"partner_name": r["partner_name"],
"prices": json.loads(r["prices"] or "{}"),
}
for r in conn.execute(sql, [limit, offset])
]
@app.post("/match", summary="Ручное сопоставление позиции со справочником")
def manual_match(item_id: str, service_id: str):
return {"item_id": item_id, "service_id": service_id, "status": "todo"} # TODO(D)
def manual_match(item_id: int, service_id: str):
with db() as conn:
if not conn.execute("SELECT 1 FROM service WHERE service_id = ?", [service_id]).fetchone():
raise HTTPException(404, "услуга справочника не найдена")
cur = conn.execute(
"UPDATE price_item SET service_id = ?, map_method = 'manual', map_confidence = 1.0 WHERE item_id = ?",
[service_id, item_id],
)
conn.commit()
if cur.rowcount == 0:
raise HTTPException(404, "позиция не найдена")
return {"item_id": item_id, "service_id": service_id, "status": "matched"}
@app.get("/stats", response_model=Stats, summary="Сводка качества обработки")
def stats():
return Stats() # TODO(D): реальные метрики из БД
report = Path(DB_PATH).parent / "quality_report.json"
if report.exists():
data = json.loads(report.read_text(encoding="utf-8"))
return Stats(
documents_total=data.get("documents", 0),
documents_done=data.get("documents", 0),
items_total=data.get("items", 0),
auto_matched_pct=data.get("auto_matched_pct", 0.0),
unmatched_total=data.get("unmatched", 0),
)
return Stats()
@app.get("/health", summary="Проверка живости")
def health():
return {"status": "ok"}
return {"status": "ok", "db": Path(DB_PATH).exists()}
+9 -1
View File
@@ -64,6 +64,7 @@ def load_dictionary(path: str | Path) -> list[Service]:
raise ValueError(f"В справочнике нет колонки Name_ru. Заголовки: {header}")
services: list[Service] = []
seen_ids: dict[str, int] = {}
for row in rows:
raw_name = row[name_idx]
if not raw_name or not str(raw_name).strip():
@@ -74,8 +75,15 @@ def load_dictionary(path: str | Path) -> list[Service]:
if tar and not TARIFICATOR_RE.match(tar):
tar = None # отбрасываем мусорные коды, оставляем только валидный формат
# «<ID>-<Code>» иногда повторяется в справочнике — гарантируем уникальность.
sid = f"{row[col['ID']]}-{row[col['Code']]}"
repeat = seen_ids.get(sid, 0)
seen_ids[sid] = repeat + 1
if repeat:
sid = f"{sid}#{repeat}"
service = Service(
service_id=f"{row[col['ID']]}-{row[col['Code']]}",
service_id=sid,
specialty=str(row[col["Специальность"]]).strip() if row[col["Специальность"]] else "",
name_ru=str(raw_name).strip(),
tarificator_code=tar,
+7 -3
View File
@@ -18,8 +18,12 @@ from etl.extractors import readers # noqa: E402
from etl.extractors.grid import extract_rows_from_grid # noqa: E402
def extract(path: str) -> ParsedDocument:
"""Разобрать один файл прайса в структурированный ParsedDocument."""
def extract(path: str, use_vision: bool = True) -> ParsedDocument:
"""Разобрать один файл прайса в структурированный ParsedDocument.
use_vision=False отключает добор через Gemini Vision (быстрый прогон без
обращений к API), даже если ключ задан.
"""
name = Path(path).name
fmt = readers.detect_format(path)
doc = ParsedDocument(file_name=name, file_format=fmt, partner_name=readers.partner_from_name(name))
@@ -55,7 +59,7 @@ def extract(path: str) -> ParsedDocument:
doc.parse_log.append(f"ошибка разбора таблицы: {type(exc).__name__}: {exc}")
# Трудные PDF (скан, битый слой, нулевой разбор) добираем через Gemini Vision.
needs_vision = fmt == "pdf" and (doc.file_format == "scan_pdf" or not doc.rows)
needs_vision = use_vision and fmt == "pdf" and (doc.file_format == "scan_pdf" or not doc.rows)
if needs_vision and os.environ.get("GEMINI_API_KEY"):
try:
from etl.extractors.vision import extract_with_vision
+137
View File
@@ -0,0 +1,137 @@
"""Конвейер обработки архива: извлечение → нормализация → валидация → SQLite.
Для MVP данные складываются в SQLite (разворачивается без Docker), а боевая схема
PostgreSQL + pgvector лежит в db/migrations. Здесь же — дедупликация партнёров,
версионирование цен (последний прайс активен, старые архивируются) и сбор отчёта о
качестве для дашборда и сдачи.
"""
from __future__ import annotations
import json
import sqlite3
import sys
import uuid
from collections import Counter
from datetime import date
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
from etl.dictionary import load_dictionary, normalize_name
from etl.extractors import extract
from etl.normalize import Matcher
from etl.validate import validate_item
SCHEMA = """
CREATE TABLE IF NOT EXISTS service (
service_id TEXT PRIMARY KEY, specialty TEXT, name_ru TEXT, name_norm TEXT, tarificator_code TEXT
);
CREATE TABLE IF NOT EXISTS partner (
partner_id TEXT PRIMARY KEY, name TEXT, name_norm TEXT UNIQUE, city TEXT
);
CREATE TABLE IF NOT EXISTS price_document (
doc_id TEXT PRIMARY KEY, partner_id TEXT, file_name TEXT, file_format TEXT,
effective_date TEXT, parse_status TEXT, rows_count INTEGER
);
CREATE TABLE IF NOT EXISTS price_item (
item_id INTEGER PRIMARY KEY AUTOINCREMENT, doc_id TEXT, partner_id TEXT,
service_name_raw TEXT, service_code_source TEXT, service_id TEXT,
prices TEXT, price_resident REAL, price_nonresident REAL, unit TEXT,
effective_date TEXT, map_method TEXT, map_confidence REAL, status TEXT, is_active INTEGER
);
CREATE INDEX IF NOT EXISTS price_item_service ON price_item(service_id);
CREATE INDEX IF NOT EXISTS price_item_partner ON price_item(partner_id);
"""
def _open(db_path: str) -> sqlite3.Connection:
conn = sqlite3.connect(db_path)
conn.executescript(SCHEMA)
for table in ("price_item", "price_document", "partner", "service"):
conn.execute(f"DELETE FROM {table}") # пересборка с нуля при каждом прогоне
return conn
def run(data_dir: str, db_path: str, dict_path: str, embedder=None, use_vision: bool = False,
today: date | None = None) -> dict:
"""Прогнать весь архив в SQLite и вернуть отчёт о качестве."""
today = today or date.today()
services = load_dictionary(dict_path)
matcher = Matcher(services, embedder=embedder)
conn = _open(db_path)
conn.executemany(
"INSERT INTO service VALUES (?,?,?,?,?)",
[(s.service_id, s.specialty, s.name_ru, s.name_norm, s.tarificator_code) for s in services],
)
partners: dict[str, str] = {}
staged: list[tuple] = [] # (doc_id, partner_id, raw, code, prices, unit, eff_iso)
report: Counter = Counter()
for path in sorted(Path(data_dir).glob("*")):
doc = extract(str(path), use_vision=use_vision)
partner_name = doc.partner_name or path.stem
partner_norm = normalize_name(partner_name)
partner_id = partners.get(partner_norm)
if partner_id is None:
partner_id = str(uuid.uuid4())
partners[partner_norm] = partner_id
conn.execute("INSERT INTO partner VALUES (?,?,?,?)", (partner_id, partner_name, partner_norm, None))
doc_id = str(uuid.uuid4())
eff_iso = doc.effective_date.isoformat() if doc.effective_date else None
conn.execute(
"INSERT INTO price_document VALUES (?,?,?,?,?,?,?)",
(doc_id, partner_id, doc.file_name, doc.file_format, eff_iso, "done", len(doc.rows)),
)
report["documents"] += 1
for row in doc.rows:
staged.append((doc_id, partner_id, row.service_name_raw, row.service_code_source,
row.prices, row.unit, eff_iso))
# Нормализация одной пачкой (эмбеддинги считаются разом — быстрее и дешевле).
matches = matcher.match_batch([s[2] for s in staged], [s[3] for s in staged])
for (doc_id, partner_id, raw, code, prices, unit, eff_iso), match in zip(staged, matches, strict=True):
status, _flags = validate_item(raw, prices, effective_date=None, today=today)
if match.service_id:
report["matched"] += 1
report[f"method_{match.method}"] += 1
conn.execute(
"""INSERT INTO price_item
(doc_id, partner_id, service_name_raw, service_code_source, service_id, prices,
price_resident, price_nonresident, unit, effective_date, map_method, map_confidence,
status, is_active)
VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,1)""",
(doc_id, partner_id, raw, code, match.service_id, json.dumps(prices, ensure_ascii=False),
prices.get("resident"), prices.get("nonresident"), unit, eff_iso, match.method,
round(match.confidence, 3), status),
)
report["items"] += 1
# Версионирование: если у партнёра по той же услуге есть более свежий прайс — старые архивируем.
conn.execute(
"""UPDATE price_item SET is_active = 0
WHERE effective_date IS NOT NULL AND EXISTS (
SELECT 1 FROM price_item b
WHERE b.partner_id = price_item.partner_id
AND b.service_name_raw = price_item.service_name_raw
AND b.effective_date > price_item.effective_date)"""
)
conn.commit()
total = report["items"]
summary = {
"documents": report["documents"],
"partners": len(partners),
"items": total,
"auto_matched": report["matched"],
"auto_matched_pct": round(100 * report["matched"] / total, 1) if total else 0.0,
"unmatched": total - report["matched"],
"by_method": {
method: report[f"method_{method}"] for method in ("code", "exact", "embedding", "fuzzy", None)
},
}
conn.close()
return summary
+27
View File
@@ -0,0 +1,27 @@
"""Запуск конвейера на архиве: data/raw → SQLite + отчёт о качестве.
Флаг --vision включает добор трудных PDF через Gemini Vision (медленно).
"""
import json
import sys
from datetime import date
from pathlib import Path
REPO = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(REPO))
from etl.normalize.embedding import GeminiEmbedder
from etl.pipeline import run
summary = run(
data_dir=str(REPO / "data/raw"),
db_path=str(REPO / "data/medarchive.db"),
dict_path=str(REPO / "data/reference/dictionary.xlsx"),
embedder=GeminiEmbedder(),
use_vision="--vision" in sys.argv,
today=date(2026, 6, 26),
)
(REPO / "data/quality_report.json").write_text(
json.dumps(summary, ensure_ascii=False, indent=2), encoding="utf-8"
)
print(json.dumps(summary, ensure_ascii=False, indent=2))
+137
View File
@@ -0,0 +1,137 @@
<!doctype html>
<html lang="ru">
<head>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1">
<title>MedArchive — сравнение цен на медуслуги</title>
<link rel="preconnect" href="https://fonts.bunny.net">
<link href="https://fonts.bunny.net/css?family=geologica:300,400,500,600,700&display=swap" rel="stylesheet">
<script src="https://cdn.tailwindcss.com"></script>
<script>
tailwind.config = { theme: { extend: {
colors: { nomad: { DEFAULT: '#ff4713', soft: '#ffede7', dark: '#e63e0f' },
tg: '#6b788e', lg: '#f5f7f9', ink: '#111827' },
fontFamily: { sans: ['Geologica', 'ui-sans-serif', 'system-ui', 'sans-serif'] } } } };
</script>
<style>body{font-family:'Geologica',system-ui,sans-serif}</style>
</head>
<body class="bg-[#f3f4f6] text-ink min-h-screen">
<div class="h-1 bg-nomad w-full"></div>
<header class="max-w-5xl mx-auto px-5 pt-8 pb-2 flex items-center justify-between">
<div class="flex items-center gap-2">
<span class="text-2xl font-bold">Med<span class="text-nomad">Archive</span></span>
<span class="text-xs text-tg bg-lg border border-[#ebedf0] rounded-full px-2 py-0.5">прайсы клиник-партнёров</span>
</div>
<nav class="text-sm text-tg flex gap-4">
<button onclick="show('search')" class="hover:text-ink">Поиск</button>
<button onclick="show('admin')" class="hover:text-ink">Админка</button>
</nav>
</header>
<main class="max-w-5xl mx-auto px-5 pb-16">
<!-- Поиск + сравнение -->
<section id="view-search">
<h1 class="text-3xl sm:text-4xl font-bold mt-6 leading-tight">Найдите услугу — <span class="text-nomad">сравните цены</span> в клиниках</h1>
<p class="text-tg mt-2">Единый каталог услуг и цен из архива прайсов клиник-партнёров.</p>
<div class="mt-5 flex gap-2">
<input id="q" type="text" placeholder="например: УЗИ брюшной полости, приём кардиолога, общий анализ крови"
class="flex-1 bg-white border border-[#d1d5db] rounded-lg px-4 py-3 outline-none focus:border-nomad focus:ring-2 focus:ring-nomad/40"
oninput="debouncedSearch()" autocomplete="off">
</div>
<div id="suggest" class="mt-2 bg-white border border-[#ebedf0] rounded-xl divide-y divide-[#f0f0f0] hidden shadow-sm"></div>
<div id="compare" class="mt-6"></div>
</section>
<!-- Админка -->
<section id="view-admin" class="hidden">
<h2 class="text-2xl font-bold mt-6">Дашборд обработки</h2>
<div id="stats" class="grid grid-cols-2 sm:grid-cols-4 gap-3 mt-4"></div>
<h3 class="text-lg font-semibold mt-8">Очередь ручной разметки (unmatched)</h3>
<div id="unmatched" class="mt-3 bg-white border border-[#ebedf0] rounded-xl overflow-hidden"></div>
</section>
</main>
<script>
const API = new URLSearchParams(location.search).get('api') || '/api';
const $ = (id) => document.getElementById(id);
const money = (v) => v == null ? '—' : new Intl.NumberFormat('ru-RU').format(v) + ' ₸';
function show(view) {
$('view-search').classList.toggle('hidden', view !== 'search');
$('view-admin').classList.toggle('hidden', view !== 'admin');
if (view === 'admin') loadAdmin();
}
let timer;
function debouncedSearch() { clearTimeout(timer); timer = setTimeout(search, 250); }
async function search() {
const q = $('q').value.trim();
const box = $('suggest');
if (q.length < 2) { box.classList.add('hidden'); return; }
const data = await fetch(`${API}/search?q=${encodeURIComponent(q)}&limit=8`).then(r => r.json());
const items = (data.services || []);
box.innerHTML = items.length
? items.map(s => `<button class="block w-full text-left px-4 py-2.5 hover:bg-nomad-soft"
onclick="compare('${s.service_id}', ${JSON.stringify(s.name_ru).replace(/"/g, '&quot;')})">
<span class="font-medium">${s.name_ru}</span>
<span class="text-xs text-tg ml-2">${s.specialty || ''}</span></button>`).join('')
: `<div class="px-4 py-3 text-tg text-sm">Ничего не найдено</div>`;
box.classList.remove('hidden');
}
async function compare(serviceId, name) {
$('suggest').classList.add('hidden');
const rows = await fetch(`${API}/services/${encodeURIComponent(serviceId)}/partners`).then(r => r.json());
const best = rows.reduce((m, r) => (r.price_resident_kzt != null && (m == null || r.price_resident_kzt < m)) ? r.price_resident_kzt : m, null);
$('compare').innerHTML = `
<div class="bg-white border border-[#ebedf0] rounded-2xl shadow-sm overflow-hidden">
<div class="px-5 py-4 border-b border-[#ebedf0]">
<div class="text-lg font-semibold">${name}</div>
<div class="text-sm text-tg">${rows.length} клиник(и) оказывают услугу</div>
</div>
<table class="w-full text-sm">
<thead class="bg-lg text-tg text-left">
<tr><th class="px-5 py-2.5 font-semibold">Клиника</th>
<th class="px-5 py-2.5 font-semibold text-right">Резидент</th>
<th class="px-5 py-2.5 font-semibold text-right">Нерезидент</th>
<th class="px-5 py-2.5 font-semibold text-right">Прайс от</th></tr>
</thead>
<tbody class="divide-y divide-[#f0f0f0]">
${rows.map(r => {
const cheapest = r.price_resident_kzt != null && r.price_resident_kzt === best;
return `<tr class="${cheapest ? 'bg-nomad-soft' : ''}">
<td class="px-5 py-3">${r.partner_name}${cheapest ? ' <span class="text-nomad text-xs font-semibold">· выгодно</span>' : ''}</td>
<td class="px-5 py-3 text-right tabular-nums font-medium">${money(r.price_resident_kzt)}</td>
<td class="px-5 py-3 text-right tabular-nums text-tg">${money(r.price_nonresident_kzt)}</td>
<td class="px-5 py-3 text-right text-tg text-xs">${r.effective_date || '—'}</td></tr>`;
}).join('')}
</tbody>
</table>
</div>`;
}
async function loadAdmin() {
const s = await fetch(`${API}/stats`).then(r => r.json());
const cards = [
['Документов', s.documents_total], ['Позиций', s.items_total],
['Автонормализация', s.auto_matched_pct + '%'], ['В очереди', s.unmatched_total]];
$('stats').innerHTML = cards.map(([l, v]) => `
<div class="bg-white border border-[#ebedf0] rounded-xl p-4">
<div class="text-2xl font-bold text-nomad">${v}</div>
<div class="text-xs text-tg mt-1">${l}</div></div>`).join('');
const u = await fetch(`${API}/unmatched?limit=25`).then(r => r.json());
$('unmatched').innerHTML = `<table class="w-full text-sm"><tbody class="divide-y divide-[#f0f0f0]">
${u.map(x => `<tr><td class="px-5 py-2.5">${x.service_name_raw}</td>
<td class="px-5 py-2.5 text-tg text-xs">${x.partner_name}</td>
<td class="px-5 py-2.5 text-right tabular-nums text-tg">${Object.values(x.prices)[0] != null ? money(Object.values(x.prices)[0]) : '—'}</td></tr>`).join('')}
</tbody></table>`;
}
</script>
</body>
</html>