Add PDF generation pipeline via browserless and Defuddle
Connects to browserless over CDP (playwright-core), extracts article content with Defuddle/JSDOM, renders it through buildCleanHtml, and prints a PDF on a second page. Adds an in-browser request filter as a second line of SSRF defense against redirects to private/localhost hosts, on top of assertPublicUrl's DNS check. Integration test is gated by RUN_PDF_INTEGRATION=1 (describe.runIf) so it is skipped in a normal npm run test and only runs against a real browserless instance.
This commit is contained in:
@@ -0,0 +1,17 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { generateCleanPdf } from "@/lib/clean-pdf/generate";
|
||||
|
||||
const enabled = process.env.RUN_PDF_INTEGRATION === "1";
|
||||
|
||||
describe.runIf(enabled)("generateCleanPdf (integration, нужен browserless)", () => {
|
||||
it("генерирует PDF реальной статьи", async () => {
|
||||
const { pdf, title } = await generateCleanPdf({
|
||||
url: "https://habr.com/ru/articles/942236/",
|
||||
format: "A4",
|
||||
theme: "light",
|
||||
});
|
||||
expect(pdf.length).toBeGreaterThan(20_000);
|
||||
expect(pdf.subarray(0, 5).toString()).toBe("%PDF-");
|
||||
expect(title.length).toBeGreaterThan(3);
|
||||
}, 120_000);
|
||||
});
|
||||
@@ -0,0 +1,99 @@
|
||||
import { isIP } from "node:net";
|
||||
import { chromium } from "playwright-core";
|
||||
import { JSDOM } from "jsdom";
|
||||
import { Defuddle } from "defuddle/node";
|
||||
import { assertPublicUrl, isPrivateAddress } from "@/lib/clean-pdf/ssrf";
|
||||
import { buildCleanHtml } from "@/lib/clean-pdf/template";
|
||||
|
||||
export class EmptyContentError extends Error {}
|
||||
export class RenderError extends Error {}
|
||||
|
||||
const GOTO_TIMEOUT_MS = 30_000;
|
||||
const SETTLE_TIMEOUT_MS = 10_000;
|
||||
const UA =
|
||||
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0 Safari/537.36";
|
||||
|
||||
function browserWsUrl(): string {
|
||||
const base = process.env.BROWSER_WS_URL;
|
||||
if (!base) throw new RenderError("BROWSER_WS_URL не задан");
|
||||
const token = process.env.BROWSERLESS_TOKEN;
|
||||
return token ? `${base}${base.includes("?") ? "&" : "?"}token=${token}` : base;
|
||||
}
|
||||
|
||||
export async function generateCleanPdf(opts: {
|
||||
url: string;
|
||||
format: "A4" | "Letter";
|
||||
theme: "light" | "dark";
|
||||
}): Promise<{ pdf: Buffer; title: string }> {
|
||||
const target = await assertPublicUrl(opts.url);
|
||||
|
||||
const browser = await chromium.connectOverCDP(browserWsUrl()).catch((e) => {
|
||||
throw new RenderError(`Браузер недоступен: ${e.message}`);
|
||||
});
|
||||
|
||||
try {
|
||||
const context = await browser.newContext({ userAgent: UA, viewport: { width: 1280, height: 800 } });
|
||||
// Вторая линия SSRF-обороны: страница не может дёргать приватные адреса
|
||||
// (литеральные IP и localhost; hostnames уже проверены до goto).
|
||||
await context.route("**/*", (route) => {
|
||||
try {
|
||||
const u = new URL(route.request().url());
|
||||
const host = u.hostname.replace(/^\[|\]$/g, "");
|
||||
if (
|
||||
(u.protocol !== "http:" && u.protocol !== "https:") ||
|
||||
host === "localhost" || host.endsWith(".local") || host.endsWith(".internal") ||
|
||||
(isIP(host) !== 0 && isPrivateAddress(host))
|
||||
) {
|
||||
return route.abort();
|
||||
}
|
||||
} catch {
|
||||
return route.abort();
|
||||
}
|
||||
return route.continue();
|
||||
});
|
||||
|
||||
const page = await context.newPage();
|
||||
await page.goto(target.href, { waitUntil: "domcontentloaded", timeout: GOTO_TIMEOUT_MS }).catch((e) => {
|
||||
throw new RenderError(`Страница не открылась: ${e.message}`);
|
||||
});
|
||||
await page.waitForLoadState("networkidle", { timeout: SETTLE_TIMEOUT_MS }).catch(() => {});
|
||||
|
||||
const rawHtml = await page.content();
|
||||
const pageTitle = await page.title().catch(() => "");
|
||||
|
||||
const dom = new JSDOM(rawHtml, { url: target.href });
|
||||
const article = await Defuddle(dom.window.document, target.href);
|
||||
// DefuddleResponse (node_modules/defuddle/dist/types.d.ts) типизирует
|
||||
// content/title/author/site/domain/wordCount как обязательные (не optional) поля —
|
||||
// сам объект тоже не nullable (Defuddle() не возвращает null/undefined).
|
||||
// Отсутствие контента выражается пустой строкой/нулевым wordCount, не отсутствием поля.
|
||||
if (!article.content || article.wordCount < 10) {
|
||||
throw new EmptyContentError("Не удалось выделить содержимое страницы");
|
||||
}
|
||||
|
||||
const title = article.title || pageTitle || target.hostname;
|
||||
const cleanHtml = buildCleanHtml({
|
||||
title,
|
||||
author: article.author || undefined,
|
||||
site: article.site || article.domain || undefined,
|
||||
url: target.href,
|
||||
contentHtml: article.content,
|
||||
theme: opts.theme,
|
||||
});
|
||||
|
||||
const pdfPage = await context.newPage();
|
||||
await pdfPage.setContent(cleanHtml, { waitUntil: "networkidle", timeout: SETTLE_TIMEOUT_MS }).catch(() => {});
|
||||
// page.pdf() в playwright-core не принимает опцию timeout (проверено по
|
||||
// node_modules/playwright-core/types/types.d.ts) — брифовский вариант с
|
||||
// timeout здесь не компилировался, поле убрано.
|
||||
const pdf = await pdfPage.pdf({
|
||||
format: opts.format,
|
||||
printBackground: true,
|
||||
margin: { top: "15mm", bottom: "18mm", left: "15mm", right: "15mm" },
|
||||
});
|
||||
|
||||
return { pdf, title };
|
||||
} finally {
|
||||
await browser.close().catch(() => {});
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user