#!/usr/bin/env node /** * CLI tool to generate markdown files from website pages. * * Usage: * node scripts/gen-web-info/index.js [options] * * Options: * --base-url Base URL to crawl (default: https://novita.ai) * --pages Comma-separated routes (e.g. /,/pricing) * --visible Run browser in visible mode (non-headless) * --help Show help */ import { chromium } from "playwright"; import TurndownService from "turndown"; import { gfm } from "@joplin/turndown-plugin-gfm"; import fs from "fs"; import path from "path"; import config from "./config.js"; function createTurndownService() { const td = new TurndownService({ headingStyle: "atx", codeBlockStyle: "fenced", bulletListMarker: "-", hr: "---", emDelimiter: "*", }); td.use(gfm); // Remove empty links and button-like anchors td.addRule("cleanLinks", { filter: (node) => { if (node.tagName !== "A") return false; const href = node.getAttribute("href") || ""; return href === "#" || href === "" || href.startsWith("javascript:"); }, replacement: (content) => content, }); // Keep meaningful link text, convert relative URLs to absolute td.addRule("simplifyLinks", { filter: "a", replacement: (content, node) => { const href = node.getAttribute("href") || ""; const text = content.trim(); if (!text) return ""; if (href.startsWith("http")) { return `[${text}](${href})`; } if (href.startsWith("/")) { return `[${text}](${config.baseUrl}${href})`; } return text; }, }); return td; } async function autoScroll(page) { await page.evaluate(async () => { await new Promise((resolve) => { let totalHeight = 0; const distance = 500; const maxTime = 30000; const startTime = Date.now(); const timer = setInterval(() => { const scrollHeight = document.body.scrollHeight; window.scrollBy(0, distance); totalHeight += distance; if (totalHeight >= scrollHeight || Date.now() - startTime > maxTime) { clearInterval(timer); window.scrollTo(0, 0); resolve(); } }, 200); }); }); } async function revealAllTabPanels(page) { await page.evaluate(() => { document .querySelectorAll('[role="tabpanel"][data-state="inactive"]') .forEach((el) => { el.removeAttribute("hidden"); el.style.display = ""; el.setAttribute("data-state", "active"); }); document .querySelectorAll('[role="tabpanel"][hidden]') .forEach((el) => { el.removeAttribute("hidden"); el.style.display = ""; }); }); } async function extractTabsContent(page, tabConfig, excludeSelectors) { const { tabSelector, panelSelector, waitMs = 3000 } = tabConfig; const tabs = page.locator(tabSelector); const count = await tabs.count(); if (count === 0) return ""; let combinedHtml = ""; for (let i = 0; i < count; i++) { const tabName = await tabs.nth(i).textContent(); console.log(` 📑 Tab ${i + 1}/${count}: ${tabName.trim()}`); await tabs.nth(i).click(); await page.waitForTimeout(waitMs); const panel = page.locator(panelSelector).first(); const panelExists = (await panel.count()) > 0; if (!panelExists) continue; const panelHtml = await panel.evaluate( (el, { selectors, expandFnSource }) => { const clone = el.cloneNode(true); selectors.forEach((sel) => { clone.querySelectorAll(sel).forEach((child) => child.remove()); }); const expandMergedCells = new Function( "return " + expandFnSource )(); expandMergedCells(clone); return clone.innerHTML; }, { selectors: excludeSelectors, expandFnSource: expandMergedCellsSource().toString(), } ); combinedHtml += `

${tabName.trim()}

\n${panelHtml}\n`; } return combinedHtml; } /** * Expand rowspan/colspan cells so every has a full set of s. * Must run inside page.evaluate() context (operates on DOM nodes). */ function expandMergedCellsSource() { return function expandMergedCells(root) { root.querySelectorAll("table").forEach((table) => { const rows = Array.from(table.querySelectorAll("tr")); // pending[rowIndex] = [{ cell, colIndex }] — cells to insert from previous rowspans const pending = []; rows.forEach((row, ri) => { const cells = Array.from(row.children); let colIdx = 0; // First, insert any pending cells carried over from rowspans above if (pending[ri]) { for (const { cell, colIndex } of pending[ri]) { const clone = cell.cloneNode(true); clone.removeAttribute("rowspan"); const ref = row.children[colIndex] || null; row.insertBefore(clone, ref); } } // Then scan for new rowspans in this row const updatedCells = Array.from(row.children); colIdx = 0; for (const cell of updatedCells) { const rs = parseInt(cell.getAttribute("rowspan") || "1", 10); if (rs > 1) { cell.removeAttribute("rowspan"); for (let offset = 1; offset < rs; offset++) { const targetRow = ri + offset; if (!pending[targetRow]) pending[targetRow] = []; pending[targetRow].push({ cell, colIndex: colIdx }); } } colIdx++; } }); }); }; } async function extractPageHTML(page, excludeSelectors) { return await page.evaluate( ({ selectors, expandFnSource }) => { const clone = document.body.cloneNode(true); selectors.forEach((sel) => { clone.querySelectorAll(sel).forEach((el) => el.remove()); }); // Remove hidden elements clone .querySelectorAll('[style*="display: none"], [style*="display:none"]') .forEach((el) => el.remove()); clone.querySelectorAll('[aria-hidden="true"]').forEach((el) => { if (!el.closest('[role="tabpanel"]')) { el.remove(); } }); // Expand rowspan cells so markdown tables are complete const expandMergedCells = new Function( "return " + expandFnSource )(); expandMergedCells(clone); // Remove empty divs to reduce noise const removeEmptyNodes = (root) => { root.querySelectorAll("div, span, section").forEach((el) => { if ( !el.textContent.trim() && !el.querySelector("table, ul, ol") ) { el.remove(); } }); }; removeEmptyNodes(clone); return clone.innerHTML; }, { selectors: excludeSelectors, expandFnSource: expandMergedCellsSource().toString(), } ); } function cleanMarkdown(md) { let result = md // Collapse 3+ newlines to 2 .replace(/\n{3,}/g, "\n\n") // Remove lines that are only whitespace .replace(/^\s+$/gm, "") // Remove lines that are only special chars like | or - .replace(/^\s*[|\\-]{1,3}\s*$/gm, "") // Remove excessive dashes .replace(/^-{4,}$/gm, "---") // Clean up table formatting (remove empty rows) .replace(/\|\s*\|\s*\n/g, "") // Fix links that wrap entire blocks — convert [block content](url) to block + link .replace(/\[((#{1,6}\s+.+?\n[\s\S]*?))\]$([^)]+)$/g, (_, content, __, url) => { return `${content.trim()}\n\nLink: ${url}`; }) .trim(); // Remove duplicate consecutive lines (handles carousel duplicates) const lines = result.split("\n"); const deduped = [lines[0]]; for (let i = 1; i < lines.length; i++) { const current = lines[i].trim(); const prev = lines[i - 1].trim(); if (current === prev && current !== "" && current !== "---") continue; deduped.push(lines[i]); } // Remove captcha/verification noise that might slip through const filtered = deduped.filter((line) => { const trimmed = line.trim(); // Only filter English verification/captcha noise lines return !( trimmed === "Security Verification" || trimmed === "Refresh Captcha" || trimmed === "Switch to Accessibility Verification" || trimmed === "Switch to Standard Verification" || trimmed === "Drag the puzzle piece below to complete verification" || trimmed === "AI-generated background" || trimmed.includes("Your speed has exceeded") || trimmed === "Verification error, please retry" ); }); return filtered.join("\n").trim(); } function buildMarkdownHeader(pageConfig, url) { const lines = [ `# ${pageConfig.title}`, "", `> Page URL: ${url}`, `> Generated on: ${new Date().toLocaleString("en-US", { timeZone: "UTC" })}`, ]; if (pageConfig.description) { lines.push(`> Description: ${pageConfig.description}`); } lines.push("", "---", ""); return lines.join("\n"); } async function generatePageMarkdown(browser, pageConfig, turndown) { const url = `${config.baseUrl}${pageConfig.route}`; const waitTime = pageConfig.waitAfterLoad || config.waitAfterLoad; console.log(`\n📄 Processing: ${pageConfig.title} (${url})`); const context = await browser.newContext({ viewport: { width: 1920, height: 1080 }, locale: "zh-CN", userAgent: "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36", }); const page = await context.newPage(); try { console.log(" ⏳ Loading page..."); try { await page.goto(url, { waitUntil: "networkidle", timeout: config.timeout, }); } catch (e) { if (e.message.includes("Timeout")) { console.log( " ⚠️ networkidle timeout, falling back to domcontentloaded..." ); await page.goto(url, { waitUntil: "domcontentloaded", timeout: config.timeout, }); await page.waitForTimeout(Math.max(waitTime, 10000)); } else { throw e; } } await page.waitForTimeout(waitTime); console.log(" ⏳ Scrolling to load lazy content..."); await autoScroll(page); await page.waitForTimeout(2000); // Execute per-page pre-extraction actions if (pageConfig.beforeExtract) { console.log(" ⏳ Running page-specific actions..."); await pageConfig.beforeExtract(page); await page.waitForTimeout(2000); } let markdown; if (pageConfig.tabExtraction) { // Tab-based extraction: extract non-tab content + each tab separately console.log(" ⏳ Extracting tab content..."); // First, extract non-tab static content by removing tablist + tabpanels from the page const staticHtml = await page.evaluate( (selectors) => { const clone = document.body.cloneNode(true); selectors.forEach((sel) => { clone.querySelectorAll(sel).forEach((el) => el.remove()); }); // Remove all tab-related elements from static extraction clone .querySelectorAll( '[role="tablist"], [role="tabpanel"], [role="tab"]' ) .forEach((el) => el.remove()); // Remove hidden elements clone .querySelectorAll( '[style*="display: none"], [style*="display:none"]' ) .forEach((el) => el.remove()); return clone.innerHTML; }, config.excludeSelectors ); // Then extract each tab's content const tabsHtml = await extractTabsContent( page, pageConfig.tabExtraction, config.excludeSelectors ); const fullHtml = staticHtml + "\n" + tabsHtml; markdown = turndown.turndown(fullHtml); markdown = cleanMarkdown(markdown); } else { // Standard extraction: reveal hidden panels and extract everything await revealAllTabPanels(page); await page.waitForTimeout(500); console.log(" ⏳ Extracting content..."); const html = await extractPageHTML(page, config.excludeSelectors); markdown = turndown.turndown(html); markdown = cleanMarkdown(markdown); } const header = buildMarkdownHeader(pageConfig, url); return header + markdown; } finally { await context.close(); } } function parseArgs() { const args = process.argv.slice(2); const options = { baseUrl: config.baseUrl, pages: null, headless: config.headless, }; for (let i = 0; i < args.length; i++) { switch (args[i]) { case "--base-url": options.baseUrl = args[++i]; break; case "--pages": options.pages = args[++i].split(",").map((s) => s.trim()); break; case "--visible": options.headless = false; break; case "--help": console.log(` Generate markdown files from website pages for knowledge base sync. Usage: node scripts/gen-web-info/index.js [options] Options: --base-url Base URL to crawl (default: ${config.baseUrl}) Can also be set via WEB_INFO_BASE_URL env var --pages Comma-separated routes to generate Example: --pages /,/pricing --visible Run browser in visible mode (for debugging) --help Show this help Configured pages: ${config.pages.map((p) => ` ${p.route.padEnd(20)} → ${p.filename}`).join("\n")} Output directory: ${config.outputDir} `); process.exit(0); } } return options; } async function main() { const options = parseArgs(); config.baseUrl = options.baseUrl; let selectedPages = config.pages; if (options.pages) { selectedPages = config.pages.filter((p) => options.pages.includes(p.route)); if (selectedPages.length === 0) { console.error( `❌ No matching pages found for: ${options.pages.join(", ")}` ); console.error( ` Available routes: ${config.pages.map((p) => p.route).join(", ")}` ); process.exit(1); } } if (!fs.existsSync(config.outputDir)) { fs.mkdirSync(config.outputDir, { recursive: true }); } console.log("🚀 Generating web info markdown files..."); console.log(` Base URL: ${config.baseUrl}`); console.log(` Output: ${config.outputDir}`); console.log( ` Pages: ${selectedPages.map((p) => p.route).join(", ")}` ); const turndown = createTurndownService(); const browser = await chromium.launch({ headless: options.headless }); try { const results = []; for (const pageConfig of selectedPages) { try { const markdown = await generatePageMarkdown( browser, pageConfig, turndown ); const outputPath = path.join(config.outputDir, pageConfig.filename); fs.writeFileSync(outputPath, markdown, "utf-8"); const lineCount = markdown.split("\n").length; const charCount = markdown.length; console.log( ` ✅ ${pageConfig.filename} (${lineCount} lines, ${charCount} chars)` ); results.push({ page: pageConfig.route, file: pageConfig.filename, success: true, }); } catch (err) { console.error(` ❌ ${pageConfig.filename}: ${err.message}`); results.push({ page: pageConfig.route, file: pageConfig.filename, success: false, error: err.message, }); } } console.log("\n📊 Summary:"); const succeeded = results.filter((r) => r.success).length; const failed = results.filter((r) => !r.success).length; console.log( ` ${succeeded} succeeded, ${failed} failed, ${results.length} total` ); if (failed > 0) { console.log(" Failed pages:"); results .filter((r) => !r.success) .forEach((r) => console.log(` - ${r.page}: ${r.error}`)); process.exit(1); } console.log(`\n🎉 All done! Files written to ${config.outputDir}/`); } finally { await browser.close(); } } main().catch((err) => { console.error("❌ Fatal error:", err); process.exit(1); });

​What’s New

​Hugging Face OpenEnv Integration

What’s New

Hugging Face OpenEnv Integration