Add PDF generation pipeline via browserless and Defuddle
Connects to browserless over CDP (playwright-core), extracts article content with Defuddle/JSDOM, renders it through buildCleanHtml, and prints a PDF on a second page. Adds an in-browser request filter as a second line of SSRF defense against redirects to private/localhost hosts, on top of assertPublicUrl's DNS check. Integration test is gated by RUN_PDF_INTEGRATION=1 (describe.runIf) so it is skipped in a normal npm run test and only runs against a real browserless instance.
This commit is contained in:
@@ -0,0 +1,17 @@
|
|||||||
|
import { describe, expect, it } from "vitest";
|
||||||
|
import { generateCleanPdf } from "@/lib/clean-pdf/generate";
|
||||||
|
|
||||||
|
const enabled = process.env.RUN_PDF_INTEGRATION === "1";
|
||||||
|
|
||||||
|
describe.runIf(enabled)("generateCleanPdf (integration, нужен browserless)", () => {
|
||||||
|
it("генерирует PDF реальной статьи", async () => {
|
||||||
|
const { pdf, title } = await generateCleanPdf({
|
||||||
|
url: "https://habr.com/ru/articles/942236/",
|
||||||
|
format: "A4",
|
||||||
|
theme: "light",
|
||||||
|
});
|
||||||
|
expect(pdf.length).toBeGreaterThan(20_000);
|
||||||
|
expect(pdf.subarray(0, 5).toString()).toBe("%PDF-");
|
||||||
|
expect(title.length).toBeGreaterThan(3);
|
||||||
|
}, 120_000);
|
||||||
|
});
|
||||||
@@ -0,0 +1,99 @@
|
|||||||
|
import { isIP } from "node:net";
|
||||||
|
import { chromium } from "playwright-core";
|
||||||
|
import { JSDOM } from "jsdom";
|
||||||
|
import { Defuddle } from "defuddle/node";
|
||||||
|
import { assertPublicUrl, isPrivateAddress } from "@/lib/clean-pdf/ssrf";
|
||||||
|
import { buildCleanHtml } from "@/lib/clean-pdf/template";
|
||||||
|
|
||||||
|
export class EmptyContentError extends Error {}
|
||||||
|
export class RenderError extends Error {}
|
||||||
|
|
||||||
|
const GOTO_TIMEOUT_MS = 30_000;
|
||||||
|
const SETTLE_TIMEOUT_MS = 10_000;
|
||||||
|
const UA =
|
||||||
|
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0 Safari/537.36";
|
||||||
|
|
||||||
|
function browserWsUrl(): string {
|
||||||
|
const base = process.env.BROWSER_WS_URL;
|
||||||
|
if (!base) throw new RenderError("BROWSER_WS_URL не задан");
|
||||||
|
const token = process.env.BROWSERLESS_TOKEN;
|
||||||
|
return token ? `${base}${base.includes("?") ? "&" : "?"}token=${token}` : base;
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function generateCleanPdf(opts: {
|
||||||
|
url: string;
|
||||||
|
format: "A4" | "Letter";
|
||||||
|
theme: "light" | "dark";
|
||||||
|
}): Promise<{ pdf: Buffer; title: string }> {
|
||||||
|
const target = await assertPublicUrl(opts.url);
|
||||||
|
|
||||||
|
const browser = await chromium.connectOverCDP(browserWsUrl()).catch((e) => {
|
||||||
|
throw new RenderError(`Браузер недоступен: ${e.message}`);
|
||||||
|
});
|
||||||
|
|
||||||
|
try {
|
||||||
|
const context = await browser.newContext({ userAgent: UA, viewport: { width: 1280, height: 800 } });
|
||||||
|
// Вторая линия SSRF-обороны: страница не может дёргать приватные адреса
|
||||||
|
// (литеральные IP и localhost; hostnames уже проверены до goto).
|
||||||
|
await context.route("**/*", (route) => {
|
||||||
|
try {
|
||||||
|
const u = new URL(route.request().url());
|
||||||
|
const host = u.hostname.replace(/^\[|\]$/g, "");
|
||||||
|
if (
|
||||||
|
(u.protocol !== "http:" && u.protocol !== "https:") ||
|
||||||
|
host === "localhost" || host.endsWith(".local") || host.endsWith(".internal") ||
|
||||||
|
(isIP(host) !== 0 && isPrivateAddress(host))
|
||||||
|
) {
|
||||||
|
return route.abort();
|
||||||
|
}
|
||||||
|
} catch {
|
||||||
|
return route.abort();
|
||||||
|
}
|
||||||
|
return route.continue();
|
||||||
|
});
|
||||||
|
|
||||||
|
const page = await context.newPage();
|
||||||
|
await page.goto(target.href, { waitUntil: "domcontentloaded", timeout: GOTO_TIMEOUT_MS }).catch((e) => {
|
||||||
|
throw new RenderError(`Страница не открылась: ${e.message}`);
|
||||||
|
});
|
||||||
|
await page.waitForLoadState("networkidle", { timeout: SETTLE_TIMEOUT_MS }).catch(() => {});
|
||||||
|
|
||||||
|
const rawHtml = await page.content();
|
||||||
|
const pageTitle = await page.title().catch(() => "");
|
||||||
|
|
||||||
|
const dom = new JSDOM(rawHtml, { url: target.href });
|
||||||
|
const article = await Defuddle(dom.window.document, target.href);
|
||||||
|
// DefuddleResponse (node_modules/defuddle/dist/types.d.ts) типизирует
|
||||||
|
// content/title/author/site/domain/wordCount как обязательные (не optional) поля —
|
||||||
|
// сам объект тоже не nullable (Defuddle() не возвращает null/undefined).
|
||||||
|
// Отсутствие контента выражается пустой строкой/нулевым wordCount, не отсутствием поля.
|
||||||
|
if (!article.content || article.wordCount < 10) {
|
||||||
|
throw new EmptyContentError("Не удалось выделить содержимое страницы");
|
||||||
|
}
|
||||||
|
|
||||||
|
const title = article.title || pageTitle || target.hostname;
|
||||||
|
const cleanHtml = buildCleanHtml({
|
||||||
|
title,
|
||||||
|
author: article.author || undefined,
|
||||||
|
site: article.site || article.domain || undefined,
|
||||||
|
url: target.href,
|
||||||
|
contentHtml: article.content,
|
||||||
|
theme: opts.theme,
|
||||||
|
});
|
||||||
|
|
||||||
|
const pdfPage = await context.newPage();
|
||||||
|
await pdfPage.setContent(cleanHtml, { waitUntil: "networkidle", timeout: SETTLE_TIMEOUT_MS }).catch(() => {});
|
||||||
|
// page.pdf() в playwright-core не принимает опцию timeout (проверено по
|
||||||
|
// node_modules/playwright-core/types/types.d.ts) — брифовский вариант с
|
||||||
|
// timeout здесь не компилировался, поле убрано.
|
||||||
|
const pdf = await pdfPage.pdf({
|
||||||
|
format: opts.format,
|
||||||
|
printBackground: true,
|
||||||
|
margin: { top: "15mm", bottom: "18mm", left: "15mm", right: "15mm" },
|
||||||
|
});
|
||||||
|
|
||||||
|
return { pdf, title };
|
||||||
|
} finally {
|
||||||
|
await browser.close().catch(() => {});
|
||||||
|
}
|
||||||
|
}
|
||||||
Reference in New Issue
Block a user