From 49214a113e57de8b0662b78555a56f794e294519 Mon Sep 17 00:00:00 2001 From: Teng Lin Date: Sun, 8 Feb 2026 12:29:06 -0500 Subject: [PATCH 1/2] refactor: decompose monolithic modules + sliding-window crawler MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Decompose content-extractors.ts (1281 → ~400 lines) into 7 focused sub-modules: metadata, readability, selector, json-ld, next-data, text-density, and RSC extractors. Barrel re-exports preserve all existing import paths. - Decompose http-fetch.ts (1074 → ~700 lines) by extracting WP REST API logic into wp-rest-api.ts and Next.js data route logic into next-data-route.ts. - Replace crawler batch-wait (Promise.all) with sliding-window concurrency (Promise.race + Map) for better throughput with non-uniform fetch times. - Promote RequestContext interface to shared fetch/types.ts. - Eliminate duplicate htmlToText by importing shared utility. - Extract attachRawHtml helper to reduce repeated keepRawHtml patterns. All 683 tests pass unchanged — no test modifications needed. Co-Authored-By: Claude Opus 4.6 --- src/crawl/crawler.ts | 73 +- src/extract/content-extractors.ts | 968 ++------------------------ src/extract/json-ld-extractor.ts | 177 +++++ src/extract/metadata-extractors.ts | 78 +++ src/extract/next-data-extractor.ts | 419 +++++++++++ src/extract/readability-extractor.ts | 70 ++ src/extract/rsc-extractor.ts | 120 ++++ src/extract/selector-extractor.ts | 86 +++ src/extract/text-density-extractor.ts | 44 ++ src/fetch/http-fetch.ts | 440 +----------- src/fetch/next-data-route.ts | 74 ++ src/fetch/types.ts | 8 + src/fetch/wp-rest-api.ts | 308 ++++++++ 13 files changed, 1500 insertions(+), 1365 deletions(-) create mode 100644 src/extract/json-ld-extractor.ts create mode 100644 src/extract/metadata-extractors.ts create mode 100644 src/extract/next-data-extractor.ts create mode 100644 src/extract/readability-extractor.ts create mode 100644 src/extract/rsc-extractor.ts create mode 100644 src/extract/selector-extractor.ts create mode 100644 src/extract/text-density-extractor.ts create mode 100644 src/fetch/next-data-route.ts create mode 100644 src/fetch/wp-rest-api.ts diff --git a/src/crawl/crawler.ts b/src/crawl/crawler.ts index 861962e..6524b02 100644 --- a/src/crawl/crawler.ts +++ b/src/crawl/crawler.ts @@ -147,19 +147,6 @@ interface ProcessOptions { onBlocked: () => void; } -/** - * Collect up to `count` entries from the frontier. - */ -function collectBatch(frontier: UrlFrontier, count: number): { url: string; depth: number }[] { - const batch: { url: string; depth: number }[] = []; - while (batch.length < count && frontier.hasMore()) { - const entry = frontier.next(); - if (!entry) break; - batch.push(entry); - } - return batch; -} - /** * Check if a URL is allowed by robots.txt Disallow rules. * Returns false (blocked) if the URL cannot be parsed. @@ -173,8 +160,10 @@ function isUrlAllowed(url: string, disallowPaths: string[]): boolean { } /** - * Process frontier entries with concurrency control. - * When discoverLinks is true, extracts links from fetched pages for BFS crawling. + * Process frontier entries with sliding-window concurrency control. + * Uses a Map-based inflight tracker so that each completed request + * immediately frees a slot for the next URL, rather than waiting + * for the entire batch to finish. */ async function* processFrontier( frontier: UrlFrontier, @@ -182,19 +171,25 @@ async function* processFrontier( ): AsyncGenerator { const { concurrency, delay, disallowPaths, options, discoverLinks, onResult, onBlocked } = opts; - while (frontier.hasMore()) { - const batch = collectBatch(frontier, concurrency); - if (batch.length === 0) break; + let nextId = 0; + const inflight = new Map>(); + + function enqueue(): void { + while (inflight.size < concurrency && frontier.hasMore()) { + const entry = frontier.next(); + if (!entry) break; + const id = nextId++; + + const promise = (async () => { + if (delay > 0) await new Promise((r) => setTimeout(r, delay)); - const results = await Promise.all( - batch.map(async (entry) => { if (!isUrlAllowed(entry.url, disallowPaths)) { logger.debug({ url: entry.url }, 'Blocked by robots.txt'); onBlocked(); - return null; + return { id, result: null }; } - const result = await httpFetch(entry.url, { + const fetchResult = await httpFetch(entry.url, { preset: options.preset, timeout: options.timeout, targetSelector: options.targetSelector, @@ -204,24 +199,32 @@ async function* processFrontier( cookies: options.cookies, }); - if (discoverLinks && result.success && result.rawHtml) { - frontier.addAll(extractLinks(result.rawHtml, entry.url), entry.depth + 1); - result.rawHtml = null; + if (discoverLinks && fetchResult.success && fetchResult.rawHtml) { + frontier.addAll(extractLinks(fetchResult.rawHtml, entry.url), entry.depth + 1); + fetchResult.rawHtml = null; } - return { ...result, depth: entry.depth } as CrawlResult; - }) - ); + return { id, result: { ...fetchResult, depth: entry.depth } as CrawlResult }; + })(); - for (const result of results) { - if (result) { - onResult(result.success); - yield result; - } + inflight.set(id, promise); } + } + + // Fill initial window + enqueue(); - if (delay > 0) { - await new Promise((resolve) => setTimeout(resolve, delay)); + while (inflight.size > 0) { + // Race all inflight promises — settled includes the id for Map deletion + const settled = await Promise.race(inflight.values()); + inflight.delete(settled.id); + + if (settled.result) { + onResult(settled.result.success); + yield settled.result; } + + // Refill window + enqueue(); } } diff --git a/src/extract/content-extractors.ts b/src/extract/content-extractors.ts index 5f90e77..3b35caa 100644 --- a/src/extract/content-extractors.ts +++ b/src/extract/content-extractors.ts @@ -1,84 +1,59 @@ /** * DOM-based content extraction strategies - * These are synchronous functions that extract content from parsed HTML documents + * + * Slim orchestrator: individual strategies live in dedicated sub-modules. + * All public symbols are re-exported so existing import paths keep working. */ import { parseHTML } from 'linkedom'; -import { Readability } from '@mozilla/readability'; -import { extractContent } from '@wrtnlabs/web-content-extractor'; import { type ExtractionResult, type SelectorOptions, MIN_CONTENT_LENGTH, GOOD_CONTENT_LENGTH, - DEFAULT_EXCERPT_LENGTH, } from './types.js'; import { htmlToMarkdown } from './markdown.js'; -import { getNestedValue, meetsThreshold, sanitizeHtml } from './utils.js'; -import { sitePreferJsonLd, siteUseNextData, getSiteNextDataPath } from '../sites/site-config.js'; +import { meetsThreshold } from './utils.js'; +import { sitePreferJsonLd, siteUseNextData } from '../sites/site-config.js'; import { tryNuxtPayloadExtraction } from './nuxt-payload.js'; import { tryReactRouterHydrationExtraction } from './react-router-hydration.js'; import { extractMedia } from './media-extractor.js'; import { logger } from '../logger.js'; import { cleanExtractedHtml } from './content-cleanup.js'; -// Selectors for finding published time -const PUBLISHED_TIME_SELECTORS = [ - 'meta[property="article:published_time"]', - 'meta[name="pubdate"]', - 'meta[name="publishdate"]', - 'meta[name="date"]', - 'time[datetime]', -]; - -// Selectors for finding article content (from Python implementation) -const CONTENT_SELECTORS = [ - 'article', - 'main article', - '[role="main"] article', - '.article-content', - '.article-body', - '.post-content', - '.entry-content', - '.story-body', - '#article-body', - '[itemprop="articleBody"]', - 'main', - '[role="main"]', -]; - -// Elements to remove before extraction -const REMOVE_SELECTORS = [ - 'script', - 'style', - 'nav', - 'aside', - 'footer', - 'header', - 'form', - 'iframe', - '.advertisement', - '.ads', - '.social-share', - '.related-articles', - '.comments', -]; +// Sub-module imports used by the orchestrator (also re-exported below) +import { tryReadability } from './readability-extractor.js'; +import { trySelectorExtraction } from './selector-extractor.js'; +import { + tryJsonLdExtraction, + detectIsAccessibleForFree, + extractJsonLdMetadata, +} from './json-ld-extractor.js'; +import { tryTextDensityExtraction } from './text-density-extractor.js'; +import { tryNextRscExtraction } from './rsc-extractor.js'; +import { tryNextDataExtraction } from './next-data-extractor.js'; + +// ── Re-exports (preserve existing import paths) ───────────────────────── +export { + extractPublishedTime, + extractTitle, + extractSiteName, + generateExcerpt, +} from './metadata-extractors.js'; +export { tryReadability } from './readability-extractor.js'; +export { trySelectorExtraction } from './selector-extractor.js'; +export { + tryJsonLdExtraction, + detectIsAccessibleForFree, + type JsonLdMetadata, +} from './json-ld-extractor.js'; +export { tryTextDensityExtraction } from './text-density-extractor.js'; +export { tryNextRscExtraction } from './rsc-extractor.js'; +export { tryNextDataExtraction, extractNextBuildId } from './next-data-extractor.js'; /** Minimum length ratio for text-density/RSC to override Readability */ const COMPARATOR_LENGTH_RATIO = 2; -/** Minimum length for RSC text segments to be considered article content */ -const RSC_MIN_SEGMENT_LENGTH = 100; - -/** JSON-LD article types that contain extractable content */ -const ARTICLE_TYPES = [ - 'Article', - 'NewsArticle', - 'BlogPosting', - 'WebPage', - 'ReportageNewsArticle', -] as const; - /** * Priority order for candidate selection in extractFromHtml. * Adding a new strategy = add its result to the results Map + append its name here. @@ -97,331 +72,7 @@ const CANDIDATE_PRIORITY = [ /** Strategies excluded from the "good candidates" tier (too noisy at threshold). */ const GOOD_CANDIDATES_EXCLUDE = new Set(['selector']); -/** - * Extract published time from meta tags - */ -export function extractPublishedTime(document: Document): string | null { - for (const selector of PUBLISHED_TIME_SELECTORS) { - const el = document.querySelector(selector); - if (el) { - const value = el.getAttribute('content') ?? el.getAttribute('datetime'); - if (value) return value; - } - } - return null; -} - -/** - * Extract title from document - */ -export function extractTitle(document: Document): string | null { - // Try og:title first - const ogTitle = document.querySelector('meta[property="og:title"]'); - if (ogTitle) { - const content = ogTitle.getAttribute('content'); - if (content) return content; - } - - // Try title tag - const titleEl = document.querySelector('title'); - if (titleEl) { - let title = titleEl.textContent?.trim() ?? ''; - // Clean up common suffixes (e.g., "Article Title - Site Name") - title = title.split(/\s*[-|–—]\s*/)[0].trim(); - if (title) return title; - } - - // Try h1 - const h1 = document.querySelector('h1'); - if (h1) { - return h1.textContent?.trim() ?? null; - } - - return null; -} - -/** - * Extract site name from document - */ -export function extractSiteName(document: Document): string | null { - return document.querySelector('meta[property="og:site_name"]')?.getAttribute('content') ?? null; -} - -/** - * Generate excerpt from text content if not already provided - */ -export function generateExcerpt(excerpt: string | null, textContent: string | null): string | null { - if (excerpt) return excerpt; - if (!textContent) return null; - - const trimmed = textContent.trim(); - if (!trimmed) return null; - - return trimmed.length > DEFAULT_EXCERPT_LENGTH - ? trimmed.slice(0, DEFAULT_EXCERPT_LENGTH) + '...' - : trimmed; -} - -/** - * Build an ExtractionResult from a Readability article and the original document. - */ -function buildReadabilityResult( - article: ReturnType['parse']>, - document: Document, - method: string -): ExtractionResult | null { - if (!article?.textContent || article.textContent.length < MIN_CONTENT_LENGTH) { - return null; - } - - return { - title: article.title ?? extractTitle(document), - byline: article.byline ?? null, - content: article.content ?? null, - textContent: article.textContent ?? null, - excerpt: generateExcerpt(article.excerpt ?? null, article.textContent ?? null), - siteName: article.siteName ?? extractSiteName(document), - publishedTime: extractPublishedTime(document) ?? article.publishedTime ?? null, - lang: article.lang ?? null, - method, - }; -} - -/** - * Strategy 1: Extract using Mozilla Readability - * Tries strict mode first, then retries with charThreshold: 100 for unusual DOM structures. - */ -export function tryReadability(document: Document, url: string): ExtractionResult | null { - try { - // Strict pass (default charThreshold of 500) - const clone = document.cloneNode(true) as Document; - const strictResult = buildReadabilityResult( - new Readability(clone).parse(), - document, - 'readability' - ); - if (strictResult) return strictResult; - - // Relaxed pass — lower charThreshold to catch unusual DOM structures - const relaxedClone = document.cloneNode(true) as Document; - const relaxedResult = buildReadabilityResult( - new Readability(relaxedClone, { charThreshold: 100 }).parse(), - document, - 'readability-relaxed' - ); - if (relaxedResult) { - logger.debug({ url }, 'Readability relaxed pass succeeded'); - } - return relaxedResult; - } catch (e) { - logger.debug({ url, error: String(e) }, 'Readability extraction failed'); - return null; - } -} - -/** - * Strategy 2: Extract using CSS selectors (like Python implementation) - */ -export function trySelectorExtraction(document: Document, url: string): ExtractionResult | null { - try { - for (const selector of CONTENT_SELECTORS) { - const el = document.querySelector(selector); - if (!el) continue; - - // Clone and clean the element - const clone = el.cloneNode(true) as Element; - - // Remove unwanted elements - for (const removeSelector of REMOVE_SELECTORS) { - const toRemove = clone.querySelectorAll(removeSelector); - toRemove.forEach((node) => node.remove()); - } - - // Get text content - const textContent = clone.textContent?.trim() ?? ''; - - if (textContent.length >= MIN_CONTENT_LENGTH) { - return { - title: extractTitle(document), - byline: null, - content: clone.innerHTML, - textContent, - excerpt: generateExcerpt(null, textContent), - siteName: extractSiteName(document), - publishedTime: extractPublishedTime(document), - lang: document.documentElement.lang || null, - method: `selector:${selector}`, - }; - } - } - } catch (e) { - logger.debug({ url, error: String(e) }, 'Selector extraction failed'); - } - - return null; -} - -/** - * Extract author name from JSON-LD author field - */ -function extractAuthorFromJsonLd(authorData: unknown): string | null { - if (typeof authorData === 'string') { - return authorData; - } - if (authorData && typeof authorData === 'object') { - const author = Array.isArray(authorData) ? authorData[0] : authorData; - if (author && typeof author === 'object') { - return (author as Record).name as string | null; - } - } - return null; -} - -/** - * Flatten a parsed JSON-LD blob into a list of individual items. - * Handles top-level arrays and @graph structures. - */ -function flattenJsonLdItems(data: unknown): Record[] { - if (!data || typeof data !== 'object') return []; - if (Array.isArray(data)) return data.flatMap(flattenJsonLdItems); - - const obj = data as Record; - if (Array.isArray(obj['@graph'])) return obj['@graph'].flatMap(flattenJsonLdItems); - - return [obj]; -} - -/** - * Check whether a JSON-LD item is a recognized article type. - */ -function isArticleType(item: Record): boolean { - const itemType = Array.isArray(item['@type']) ? item['@type'][0] : item['@type']; - return ( - typeof itemType === 'string' && - ARTICLE_TYPES.includes(itemType as (typeof ARTICLE_TYPES)[number]) - ); -} - -/** - * Parse a single JSON-LD item to extract article content - */ -function parseJsonLdItem( - item: Record -): Omit | null { - if (!isArticleType(item)) return null; - - const content = - (item.articleBody as string) ?? (item.text as string) ?? (item.description as string); - if (!content) return null; - - return { - title: (item.headline as string) ?? (item.name as string) ?? null, - byline: extractAuthorFromJsonLd(item.author), - content, - textContent: content, - excerpt: (item.description as string) ?? null, - }; -} - -/** - * Parse all JSON-LD script tags and return flattened article-candidate items. - */ -function parseJsonLdScripts(document: Document): Record[] { - const items: Record[] = []; - const scripts = document.querySelectorAll('script[type="application/ld+json"]'); - for (const script of scripts) { - try { - const data = JSON.parse(script.textContent ?? ''); - items.push(...flattenJsonLdItems(data)); - } catch { - // Skip malformed JSON - } - } - return items; -} - -/** - * Strategy 3: Extract from JSON-LD structured data - */ -export function tryJsonLdExtraction(document: Document, url: string): ExtractionResult | null { - try { - for (const item of parseJsonLdScripts(document)) { - const article = parseJsonLdItem(item); - if (article?.content && article.content.length >= MIN_CONTENT_LENGTH) { - return { - ...article, - siteName: extractSiteName(document), - publishedTime: extractPublishedTime(document), - lang: document.documentElement.lang || null, - method: 'json-ld', - }; - } - } - } catch (e) { - logger.debug({ url, error: String(e) }, 'JSON-LD extraction failed'); - } - - return null; -} - -interface JsonLdMetadata { - byline: string | null; - publishedTime: string | null; -} - -/** - * Extract metadata-only fields from JSON-LD (author, dates). - * Unlike tryJsonLdExtraction, this does NOT require articleBody to meet content threshold. - * Used for metadata composition when another strategy wins for content. - */ -function extractJsonLdMetadata(document: Document): JsonLdMetadata | null { - for (const item of parseJsonLdScripts(document)) { - if (!isArticleType(item)) continue; - - const byline = extractAuthorFromJsonLd(item.author); - const publishedTime = (item.datePublished as string) ?? (item.dateCreated as string) ?? null; - if (byline || publishedTime) { - return { byline, publishedTime }; - } - } - return null; -} - -interface SchemaOrgAccessInfo { - isAccessibleForFree: boolean; - declaredWordCount?: number; -} - -/** Parse a schema.org wordCount value (number or numeric string) into a number. */ -function parseWordCount(value: unknown): number | undefined { - if (typeof value === 'number') return value; - if (typeof value === 'string') { - const num = parseInt(value, 10); - return isNaN(num) ? undefined : num; - } - return undefined; -} - -/** - * Read the schema.org `isAccessibleForFree` property from JSON-LD structured data - * that publishers embed in their pages. Returns a result only when the field is - * explicitly set to `false`. Handles both boolean and string representations - * (e.g. `"False"` as used by some publishers). Also reads `wordCount` when present. - */ -export function detectIsAccessibleForFree(document: Document): SchemaOrgAccessInfo | null { - for (const item of parseJsonLdScripts(document)) { - if (!isArticleType(item)) continue; - if (!('isAccessibleForFree' in item)) continue; - - const raw = item.isAccessibleForFree; - const isFree = raw === true || (typeof raw === 'string' && raw.toLowerCase() === 'true'); - if (isFree) return null; - - const declaredWordCount = parseWordCount(item.wordCount); - return { isAccessibleForFree: false, declaredWordCount }; - } - return null; -} +// ── Composition & markdown helpers ────────────────────────────────────── /** * Compose best metadata from multiple extraction results and JSON-LD metadata. @@ -430,7 +81,7 @@ export function detectIsAccessibleForFree(document: Document): SchemaOrgAccessIn function composeMetadata( winner: ExtractionResult, candidates: (ExtractionResult | null)[], - jsonLdMeta: JsonLdMetadata | null + jsonLdMeta: { byline: string | null; publishedTime: string | null } | null ): ExtractionResult { const composed = { ...winner }; @@ -484,548 +135,7 @@ function withMarkdown(result: ExtractionResult): ExtractionResult { return { ...result, markdown }; } -/** - * Extract text content from HTML string - */ -function extractTextFromHtml(html: string): string { - // Wrap in full HTML structure so linkedom parses correctly - const { document } = parseHTML(`${html}`); - return document.body.textContent?.trim() ?? ''; -} - -/** - * Extract byline from an authors array - */ -function extractBylineFromAuthors(authors: unknown): string | null { - if (!Array.isArray(authors) || authors.length === 0) return null; - const names = authors - .map((author: unknown) => { - if ( - author && - typeof author === 'object' && - 'name' in author && - typeof (author as { name: unknown }).name === 'string' - ) { - return (author as { name: string }).name; - } - return null; - }) - .filter((name): name is string => name !== null && name.length > 0); - return names.length > 0 ? names.join(', ') : null; -} - -/** - * Content block with type and text/textHtml fields - */ -interface ContentBlock { - type?: string; - text?: string; - textHtml?: string; - components?: ContentBlock[]; - items?: ContentBlock[]; -} - -/** - * Extract text from an array of content blocks. - * Handles structures where body is an array of typed blocks with text/textHtml fields. - */ -function extractTextFromContentBlocks(blocks: ContentBlock[]): string { - const textParts: string[] = []; - - function processBlock(block: ContentBlock): void { - if (!block || typeof block !== 'object') return; - const blockType = typeof block.type === 'string' ? block.type.toUpperCase() : undefined; - - // Extract text from paragraph-like blocks - if (blockType === 'PARAGRAPH' || blockType === 'HEADING' || blockType === 'SUBHEADING') { - const text = block.text ?? (block.textHtml ? extractTextFromHtml(block.textHtml) : ''); - if (text) { - textParts.push(text); - textParts.push('\n\n'); - } - return; - } - - // Handle list items - if (blockType === 'UNORDERED_LIST' || blockType === 'ORDERED_LIST') { - if (Array.isArray(block.items)) { - for (const item of block.items) { - if (!item || typeof item !== 'object') continue; - const text = item.text ?? (item.textHtml ? extractTextFromHtml(item.textHtml) : ''); - if (text) { - textParts.push(`• ${text}\n`); - } - } - textParts.push('\n'); - } - return; - } - - // Recurse into nested components (e.g., INFOBOX) - if (Array.isArray(block.components)) { - for (const component of block.components) { - processBlock(component); - } - } - } - - for (const block of blocks) { - processBlock(block); - } - - return textParts.join('').trim(); -} - -/** - * Common paths where Next.js sites store article content as arrays of content blocks. - * These are probed in order when no site-specific config exists. - */ -const NEXT_DATA_CONTENT_PATHS = [ - 'props.pageProps.content.body', - 'props.pageProps.article.body', - 'props.pageProps.article.content', - 'props.pageProps.post.body', - 'props.pageProps.post.content', - 'props.pageProps.data.body', -]; - -/** - * Check if a value looks like an array of content blocks (has objects with type/text fields). - */ -function isContentBlockArray(value: unknown): value is ContentBlock[] { - if (!Array.isArray(value) || value.length === 0) return false; - // Check first few items for content block structure - const sample = value.slice(0, 5); - return sample.some( - (item) => - item && typeof item === 'object' && ('type' in item || 'text' in item || 'textHtml' in item) - ); -} - -/** - * Try to extract metadata for content block arrays from various pageProps locations. - */ -function extractContentBlockMetadata( - pageProps: Record, - document: Document -): { - title: string | null; - byline: string | null; - excerpt: string | null; - publishedTime: string | null; -} { - // Try content.headline, article.headline, post.title, etc. - const content = pageProps.content as Record | undefined; - const article = pageProps.article as Record | undefined; - const post = pageProps.post as Record | undefined; - - const title = - (content?.headline as string) ?? - (article?.headline as string) ?? - (article?.title as string) ?? - (post?.title as string) ?? - (pageProps.title as string) ?? - extractTitle(document); - - const byline = - extractBylineFromAuthors(content?.authors) ?? - extractBylineFromAuthors(article?.authors) ?? - extractBylineFromAuthors(post?.authors) ?? - (pageProps.author as { name?: string })?.name ?? - null; - - const excerpt = - (content?.description as string) ?? - (article?.description as string) ?? - (article?.excerpt as string) ?? - (post?.excerpt as string) ?? - (pageProps.description as string) ?? - null; - - const publishedTime = - (content?.datePublished as string) ?? - (article?.datePublished as string) ?? - (article?.publishedAt as string) ?? - (post?.date as string) ?? - (pageProps.publishedAt as string) ?? - extractPublishedTime(document); - - return { title, byline, excerpt, publishedTime }; -} - -/** - * Try to extract an ExtractionResult from a string body (HTML or plain text). - * Shared by both the custom-path and auto-detect branches of tryNextDataExtraction. - */ -function tryStringBodyExtraction( - content: string, - meta: Record, - document: Document -): ExtractionResult | null { - if (content.length < MIN_CONTENT_LENGTH) return null; - - const isHtml = /<(?:p|div)>/.test(content); - const sanitized = isHtml ? sanitizeHtml(content) : content; - const textContent = isHtml ? extractTextFromHtml(sanitized) : content; - if (textContent.length < MIN_CONTENT_LENGTH) return null; - - return { - title: (meta.headline as string) ?? (meta.title as string) ?? extractTitle(document), - byline: extractBylineFromAuthors(meta.authors) ?? (meta.byline as string) ?? null, - content: isHtml ? sanitized : textContent, - textContent, - excerpt: - (meta.description as string) ?? - (meta.excerpt as string) ?? - generateExcerpt(null, textContent), - siteName: extractSiteName(document), - publishedTime: - (meta.datePublished as string) ?? - (meta.publishedAt as string) ?? - extractPublishedTime(document), - lang: document.documentElement.lang || 'en', - method: isHtml ? 'next-data-html' : 'next-data', - }; -} - -/** - * Strategy 4: Extract from Next.js __NEXT_DATA__ - * Some sites embed full article content in the page props JSON. - * Supports three modes (tried in order): - * 1. Site-specific: Use nextDataPath config to specify custom JSON path - * 2. Auto-detect: Probe common content paths for content block arrays or string bodies - * 3. Default: Look for story.body.content structured blocks - */ -export function tryNextDataExtraction(document: Document, url: string): ExtractionResult | null { - try { - const script = document.querySelector('script#__NEXT_DATA__'); - if (!script?.textContent) return null; - - const data = JSON.parse(script.textContent); - const pageProps = data?.props?.pageProps; - if (!pageProps) return null; - - // Try site-specific path first - const customPath = getSiteNextDataPath(url); - if (customPath) { - const content = getNestedValue(data, customPath); - - // Handle array of content blocks - if (Array.isArray(content)) { - const textContent = extractTextFromContentBlocks( - content.filter((item): item is ContentBlock => item && typeof item === 'object') - ); - if (textContent.length >= MIN_CONTENT_LENGTH) { - logger.debug( - { url, path: customPath }, - 'Next.js custom path (array) extraction succeeded' - ); - // Extract metadata from content structure - const contentMeta = pageProps.content ?? {}; - return { - title: contentMeta.headline ?? pageProps.title ?? extractTitle(document), - byline: extractBylineFromAuthors(contentMeta.authors) ?? pageProps.author?.name ?? null, - content: textContent, - textContent, - excerpt: - contentMeta.description ?? - pageProps.teaser_body ?? - generateExcerpt(null, textContent), - siteName: extractSiteName(document), - publishedTime: - contentMeta.datePublished ?? - pageProps.publishedAt ?? - pageProps.changed_formatted ?? - extractPublishedTime(document), - lang: document.documentElement.lang || 'en', - method: 'next-data', - }; - } - } - - // Handle string content (HTML or plain text) - if (typeof content === 'string') { - // Map pageProps fields to the generic metadata shape expected by tryStringBodyExtraction - const meta: Record = { - title: pageProps.title, - byline: pageProps.author?.name ?? null, - description: pageProps.teaser_body ?? pageProps.description, - publishedAt: pageProps.publishedAt ?? pageProps.changed_formatted, - }; - const result = tryStringBodyExtraction(content, meta, document); - if (result) { - logger.debug({ url, path: customPath }, 'Next.js custom path extraction succeeded'); - return result; - } - } - } - - // Auto-detect: Probe common content paths for content block arrays or string bodies - for (const path of NEXT_DATA_CONTENT_PATHS) { - const content = getNestedValue(data, path); - if (isContentBlockArray(content)) { - const textContent = extractTextFromContentBlocks( - content.filter((item): item is ContentBlock => item && typeof item === 'object') - ); - if (textContent.length >= MIN_CONTENT_LENGTH) { - logger.debug({ url, path }, 'Next.js auto-detected content path extraction succeeded'); - const meta = extractContentBlockMetadata(pageProps as Record, document); - return { - title: meta.title ?? extractTitle(document), - byline: meta.byline, - content: textContent, - textContent, - excerpt: meta.excerpt ?? generateExcerpt(null, textContent), - siteName: extractSiteName(document), - publishedTime: meta.publishedTime ?? extractPublishedTime(document), - lang: document.documentElement.lang || 'en', - method: 'next-data', - }; - } - } - - // Handle string content (HTML or plain text) - if (typeof content === 'string') { - const parentPath = path.split('.').slice(0, -1).join('.'); - const parent = (getNestedValue(data, parentPath) ?? {}) as Record; - // Fall back to pageProps.title when parent has no title fields - const metaWithFallback = { ...parent, title: parent.title ?? pageProps.title }; - const result = tryStringBodyExtraction(content, metaWithFallback, document); - if (result) { - logger.debug({ url, path }, 'Next.js auto-detected string content extraction succeeded'); - return result; - } - } - } - - // Default: Try story.body.content structured blocks - const story = pageProps?.story; - if (!story?.body?.content) return null; - - // Extract text from the structured content - const textParts: string[] = []; - - function extractText(node: unknown): void { - if (!node || typeof node !== 'object') return; - - const obj = node as Record; - - // Handle text nodes - if (obj.type === 'text' && typeof obj.value === 'string') { - textParts.push(obj.value); - return; - } - - // Skip non-content types - const skipTypes = ['inline-newsletter', 'ad', 'related-content', 'inline-recirc']; - if (typeof obj.type === 'string' && skipTypes.includes(obj.type)) { - return; - } - - // Add newlines after paragraphs - if (obj.type === 'paragraph') { - if (Array.isArray(obj.content)) { - obj.content.forEach(extractText); - } - textParts.push('\n\n'); - return; - } - - // Recurse into content arrays - if (Array.isArray(obj.content)) { - obj.content.forEach(extractText); - } - } - - // Process all content blocks - if (Array.isArray(story.body.content)) { - story.body.content.forEach(extractText); - } - - const textContent = textParts.join('').trim(); - if (textContent.length < MIN_CONTENT_LENGTH) return null; - - // Extract author - let byline: string | null = null; - if (Array.isArray(story.authors) && story.authors.length > 0) { - byline = story.authors - .map((a: { name?: string }) => a.name) - .filter(Boolean) - .join(', '); - } - - return { - title: story.headline ?? story.title ?? extractTitle(document), - byline, - content: textContent, - textContent, - excerpt: story.abstract?.[0] ?? generateExcerpt(null, textContent), - siteName: extractSiteName(document), - publishedTime: story.publishedAt ?? extractPublishedTime(document), - lang: document.documentElement.lang || 'en', - method: 'next-data', - }; - } catch (e) { - logger.debug({ url, error: String(e) }, 'Next.js data extraction failed'); - return null; - } -} - -/** - * Strategy 5: Extract using text density analysis (CETD algorithm). - * Statistical approach based on text-to-tag density ratios per DOM node. - * Complementary to Readability's heuristic class/id scoring. - */ -export function tryTextDensityExtraction(html: string, url: string): ExtractionResult | null { - try { - const extracted = extractContent(html); - - if (extracted.content.length < MIN_CONTENT_LENGTH) { - return null; - } - - // Parse with linkedom for metadata extraction (reuses the same DOM lib as other strategies) - const { document } = parseHTML(html); - - return { - title: extractTitle(document), - byline: null, - content: extracted.contentHtmls.join(''), - textContent: extracted.content, - excerpt: generateExcerpt(extracted.description ?? null, extracted.content), - siteName: extractSiteName(document), - publishedTime: extractPublishedTime(document), - lang: document.documentElement.lang || null, - method: 'text-density', - }; - } catch (e) { - logger.debug({ url, error: String(e) }, 'Text density extraction failed'); - return null; - } -} - -/** - * Check whether a string looks like natural language (not HTML/JS). - */ -function isNaturalLanguage(text: string): boolean { - const len = text.length; - if (len === 0) return false; - - // Reject HTML-heavy content (more than 1 tag per 100 chars) - const tagCount = text.split('<').length - 1; - if (tagCount > len / 100) return false; - - // Reject JS-heavy content - if (text.includes('function(') || text.includes('function (')) return false; - const arrowCount = text.split('=>').length - 1; - if (arrowCount > 3) return false; - - // Must have word-like content: spaces between words - const spaceCount = text.split(' ').length - 1; - return spaceCount / len > 0.1; -} - -/** - * Strategy 6: Extract from Next.js RSC (React Server Components) streaming payload. - * App Router pages embed article text inside self.__next_f.push() script calls - * rather than rendering it into the DOM. - */ -export function tryNextRscExtraction(html: string, url: string): ExtractionResult | null { - try { - if (!html.includes('self.__next_f.push(')) return null; - - // Extract all self.__next_f.push([...]) calls, anchored on boundary - const pushPattern = /self\.__next_f\.push\(([\s\S]*?)\)<\/script>/g; - const chunks: string[] = []; - let match; - while ((match = pushPattern.exec(html)) !== null) { - try { - const arr = JSON.parse(match[1]); - if (Array.isArray(arr) && arr[0] === 1 && typeof arr[1] === 'string') { - chunks.push(arr[1]); - } - } catch { - // Skip malformed JSON (some chunks contain unescaped JS) - } - } - - if (chunks.length === 0) return null; - - // Concatenate all type-1 chunks into a single stream - const stream = chunks.join(''); - - // Extract text segments from RSC T markers: id:Thexlen, - const textSegments: string[] = []; - const tMarkerPattern = /[0-9a-f]+:T[0-9a-f]+,/g; - let tMatch; - const tPositions: number[] = []; - while ((tMatch = tMarkerPattern.exec(stream)) !== null) { - tPositions.push(tMatch.index + tMatch[0].length); - } - - // For each T marker, extract text until next RSC row prefix or end - const rowPrefixPattern = /\n[0-9a-f]+:[A-Z["$]/; - for (const pos of tPositions) { - const rest = stream.slice(pos); - const nextRow = rest.search(rowPrefixPattern); - const segment = nextRow === -1 ? rest : rest.slice(0, nextRow); - - if (segment.length >= RSC_MIN_SEGMENT_LENGTH && isNaturalLanguage(segment)) { - textSegments.push(segment.trim()); - } - } - - // Also check for continuation chunks (raw text with no row prefix) - for (const chunk of chunks) { - if ( - chunk.length >= RSC_MIN_SEGMENT_LENGTH && - !/^[0-9a-f]+:/.test(chunk) && - isNaturalLanguage(chunk) - ) { - textSegments.push(chunk.trim()); - } - } - - // Deduplicate (continuation chunks may overlap with T marker extraction) - const unique = [...new Set(textSegments)]; - const textContent = unique.join('\n\n'); - - if (textContent.length < MIN_CONTENT_LENGTH) return null; - - const { document } = parseHTML(html); - - return { - title: extractTitle(document), - byline: null, - content: textContent, - textContent, - excerpt: generateExcerpt(null, textContent), - siteName: extractSiteName(document), - publishedTime: extractPublishedTime(document), - lang: document.documentElement.lang || null, - method: 'next-rsc', - }; - } catch (e) { - logger.debug({ url, error: String(e) }, 'Next.js RSC extraction failed'); - return null; - } -} - -/** - * Extract the Next.js buildId from a __NEXT_DATA__ script tag. - * Returns null if the page is not a Next.js page or has no buildId. - */ -export function extractNextBuildId(document: Document): string | null { - try { - const script = document.querySelector('script#__NEXT_DATA__'); - if (!script?.textContent) return null; - const data = JSON.parse(script.textContent); - const buildId = data?.buildId; - return typeof buildId === 'string' && /^[\w-]+$/.test(buildId) ? buildId : null; - } catch { - return null; - } -} +// ── WordPress detection ───────────────────────────────────────────────── /** * Detect WordPress REST API availability from HTML. @@ -1048,6 +158,8 @@ export function detectWpRestApi(document: Document, pageUrl: string): string | n return href; } +// ── Selector application ──────────────────────────────────────────────── + /** Normalize a selector option to an array. */ function toSelectorArray(sel: string | string[] | undefined): string[] { if (!sel) return []; @@ -1098,6 +210,8 @@ export function applySelectors(html: string, selectors: SelectorOptions): string return document.toString(); } +// ── Main extraction orchestrator ──────────────────────────────────────── + /** * Multi-strategy extraction from HTML * Exported for testing and direct HTML extraction use cases diff --git a/src/extract/json-ld-extractor.ts b/src/extract/json-ld-extractor.ts new file mode 100644 index 0000000..edd0b16 --- /dev/null +++ b/src/extract/json-ld-extractor.ts @@ -0,0 +1,177 @@ +/** + * Strategy 3: Extract from JSON-LD structured data + */ +import { type ExtractionResult, MIN_CONTENT_LENGTH } from './types.js'; +import { extractPublishedTime, extractSiteName } from './metadata-extractors.js'; +import { logger } from '../logger.js'; + +/** JSON-LD article types that contain extractable content */ +const ARTICLE_TYPES = [ + 'Article', + 'NewsArticle', + 'BlogPosting', + 'WebPage', + 'ReportageNewsArticle', +] as const; + +/** + * Extract author name from JSON-LD author field + */ +function extractAuthorFromJsonLd(authorData: unknown): string | null { + if (typeof authorData === 'string') { + return authorData; + } + if (authorData && typeof authorData === 'object') { + const author = Array.isArray(authorData) ? authorData[0] : authorData; + if (author && typeof author === 'object') { + return (author as Record).name as string | null; + } + } + return null; +} + +/** + * Flatten a parsed JSON-LD blob into a list of individual items. + * Handles top-level arrays and @graph structures. + */ +function flattenJsonLdItems(data: unknown): Record[] { + if (!data || typeof data !== 'object') return []; + if (Array.isArray(data)) return data.flatMap(flattenJsonLdItems); + + const obj = data as Record; + if (Array.isArray(obj['@graph'])) return obj['@graph'].flatMap(flattenJsonLdItems); + + return [obj]; +} + +/** + * Check whether a JSON-LD item is a recognized article type. + */ +function isArticleType(item: Record): boolean { + const itemType = Array.isArray(item['@type']) ? item['@type'][0] : item['@type']; + return ( + typeof itemType === 'string' && + ARTICLE_TYPES.includes(itemType as (typeof ARTICLE_TYPES)[number]) + ); +} + +/** + * Parse a single JSON-LD item to extract article content + */ +function parseJsonLdItem( + item: Record +): Omit | null { + if (!isArticleType(item)) return null; + + const content = + (item.articleBody as string) ?? (item.text as string) ?? (item.description as string); + if (!content) return null; + + return { + title: (item.headline as string) ?? (item.name as string) ?? null, + byline: extractAuthorFromJsonLd(item.author), + content, + textContent: content, + excerpt: (item.description as string) ?? null, + }; +} + +/** + * Parse all JSON-LD script tags and return flattened article-candidate items. + */ +function parseJsonLdScripts(document: Document): Record[] { + const items: Record[] = []; + const scripts = document.querySelectorAll('script[type="application/ld+json"]'); + for (const script of scripts) { + try { + const data = JSON.parse(script.textContent ?? ''); + items.push(...flattenJsonLdItems(data)); + } catch { + // Skip malformed JSON + } + } + return items; +} + +/** + * Strategy 3: Extract from JSON-LD structured data + */ +export function tryJsonLdExtraction(document: Document, url: string): ExtractionResult | null { + try { + for (const item of parseJsonLdScripts(document)) { + const article = parseJsonLdItem(item); + if (article?.content && article.content.length >= MIN_CONTENT_LENGTH) { + return { + ...article, + siteName: extractSiteName(document), + publishedTime: extractPublishedTime(document), + lang: document.documentElement.lang || null, + method: 'json-ld', + }; + } + } + } catch (e) { + logger.debug({ url, error: String(e) }, 'JSON-LD extraction failed'); + } + + return null; +} + +export interface JsonLdMetadata { + byline: string | null; + publishedTime: string | null; +} + +/** + * Extract metadata-only fields from JSON-LD (author, dates). + * Unlike tryJsonLdExtraction, this does NOT require articleBody to meet content threshold. + * Used for metadata composition when another strategy wins for content. + */ +export function extractJsonLdMetadata(document: Document): JsonLdMetadata | null { + for (const item of parseJsonLdScripts(document)) { + if (!isArticleType(item)) continue; + + const byline = extractAuthorFromJsonLd(item.author); + const publishedTime = (item.datePublished as string) ?? (item.dateCreated as string) ?? null; + if (byline || publishedTime) { + return { byline, publishedTime }; + } + } + return null; +} + +interface SchemaOrgAccessInfo { + isAccessibleForFree: boolean; + declaredWordCount?: number; +} + +/** Parse a schema.org wordCount value (number or numeric string) into a number. */ +function parseWordCount(value: unknown): number | undefined { + if (typeof value === 'number') return value; + if (typeof value === 'string') { + const num = parseInt(value, 10); + return isNaN(num) ? undefined : num; + } + return undefined; +} + +/** + * Read the schema.org `isAccessibleForFree` property from JSON-LD structured data + * that publishers embed in their pages. Returns a result only when the field is + * explicitly set to `false`. Handles both boolean and string representations + * (e.g. `"False"` as used by some publishers). Also reads `wordCount` when present. + */ +export function detectIsAccessibleForFree(document: Document): SchemaOrgAccessInfo | null { + for (const item of parseJsonLdScripts(document)) { + if (!isArticleType(item)) continue; + if (!('isAccessibleForFree' in item)) continue; + + const raw = item.isAccessibleForFree; + const isFree = raw === true || (typeof raw === 'string' && raw.toLowerCase() === 'true'); + if (isFree) return null; + + const declaredWordCount = parseWordCount(item.wordCount); + return { isAccessibleForFree: false, declaredWordCount }; + } + return null; +} diff --git a/src/extract/metadata-extractors.ts b/src/extract/metadata-extractors.ts new file mode 100644 index 0000000..9a384df --- /dev/null +++ b/src/extract/metadata-extractors.ts @@ -0,0 +1,78 @@ +/** + * Metadata extraction helpers: title, site name, published time, excerpt. + */ +import { DEFAULT_EXCERPT_LENGTH } from './types.js'; + +// Selectors for finding published time +const PUBLISHED_TIME_SELECTORS = [ + 'meta[property="article:published_time"]', + 'meta[name="pubdate"]', + 'meta[name="publishdate"]', + 'meta[name="date"]', + 'time[datetime]', +]; + +/** + * Extract published time from meta tags + */ +export function extractPublishedTime(document: Document): string | null { + for (const selector of PUBLISHED_TIME_SELECTORS) { + const el = document.querySelector(selector); + if (el) { + const value = el.getAttribute('content') ?? el.getAttribute('datetime'); + if (value) return value; + } + } + return null; +} + +/** + * Extract title from document + */ +export function extractTitle(document: Document): string | null { + // Try og:title first + const ogTitle = document.querySelector('meta[property="og:title"]'); + if (ogTitle) { + const content = ogTitle.getAttribute('content'); + if (content) return content; + } + + // Try title tag + const titleEl = document.querySelector('title'); + if (titleEl) { + let title = titleEl.textContent?.trim() ?? ''; + // Clean up common suffixes (e.g., "Article Title - Site Name") + title = title.split(/\s*[-|–—]\s*/)[0].trim(); + if (title) return title; + } + + // Try h1 + const h1 = document.querySelector('h1'); + if (h1) { + return h1.textContent?.trim() ?? null; + } + + return null; +} + +/** + * Extract site name from document + */ +export function extractSiteName(document: Document): string | null { + return document.querySelector('meta[property="og:site_name"]')?.getAttribute('content') ?? null; +} + +/** + * Generate excerpt from text content if not already provided + */ +export function generateExcerpt(excerpt: string | null, textContent: string | null): string | null { + if (excerpt) return excerpt; + if (!textContent) return null; + + const trimmed = textContent.trim(); + if (!trimmed) return null; + + return trimmed.length > DEFAULT_EXCERPT_LENGTH + ? trimmed.slice(0, DEFAULT_EXCERPT_LENGTH) + '...' + : trimmed; +} diff --git a/src/extract/next-data-extractor.ts b/src/extract/next-data-extractor.ts new file mode 100644 index 0000000..2609d2b --- /dev/null +++ b/src/extract/next-data-extractor.ts @@ -0,0 +1,419 @@ +/** + * Strategy 4: Extract from Next.js __NEXT_DATA__ + * Some sites embed full article content in the page props JSON. + */ +import { parseHTML } from 'linkedom'; + +import { type ExtractionResult, MIN_CONTENT_LENGTH } from './types.js'; +import { + extractPublishedTime, + extractTitle, + extractSiteName, + generateExcerpt, +} from './metadata-extractors.js'; +import { getNestedValue, sanitizeHtml } from './utils.js'; +import { getSiteNextDataPath } from '../sites/site-config.js'; +import { logger } from '../logger.js'; + +/** + * Extract text content from HTML string + */ +function extractTextFromHtml(html: string): string { + // Wrap in full HTML structure so linkedom parses correctly + const { document } = parseHTML(`${html}`); + return document.body.textContent?.trim() ?? ''; +} + +/** + * Extract byline from an authors array + */ +function extractBylineFromAuthors(authors: unknown): string | null { + if (!Array.isArray(authors) || authors.length === 0) return null; + const names = authors + .map((author: unknown) => { + if ( + author && + typeof author === 'object' && + 'name' in author && + typeof (author as { name: unknown }).name === 'string' + ) { + return (author as { name: string }).name; + } + return null; + }) + .filter((name): name is string => name !== null && name.length > 0); + return names.length > 0 ? names.join(', ') : null; +} + +/** + * Content block with type and text/textHtml fields + */ +interface ContentBlock { + type?: string; + text?: string; + textHtml?: string; + components?: ContentBlock[]; + items?: ContentBlock[]; +} + +/** + * Extract text from an array of content blocks. + * Handles structures where body is an array of typed blocks with text/textHtml fields. + */ +function extractTextFromContentBlocks(blocks: ContentBlock[]): string { + const textParts: string[] = []; + + function processBlock(block: ContentBlock): void { + if (!block || typeof block !== 'object') return; + const blockType = typeof block.type === 'string' ? block.type.toUpperCase() : undefined; + + // Extract text from paragraph-like blocks + if (blockType === 'PARAGRAPH' || blockType === 'HEADING' || blockType === 'SUBHEADING') { + const text = block.text ?? (block.textHtml ? extractTextFromHtml(block.textHtml) : ''); + if (text) { + textParts.push(text); + textParts.push('\n\n'); + } + return; + } + + // Handle list items + if (blockType === 'UNORDERED_LIST' || blockType === 'ORDERED_LIST') { + if (Array.isArray(block.items)) { + for (const item of block.items) { + if (!item || typeof item !== 'object') continue; + const text = item.text ?? (item.textHtml ? extractTextFromHtml(item.textHtml) : ''); + if (text) { + textParts.push(`• ${text}\n`); + } + } + textParts.push('\n'); + } + return; + } + + // Recurse into nested components (e.g., INFOBOX) + if (Array.isArray(block.components)) { + for (const component of block.components) { + processBlock(component); + } + } + } + + for (const block of blocks) { + processBlock(block); + } + + return textParts.join('').trim(); +} + +/** + * Common paths where Next.js sites store article content as arrays of content blocks. + * These are probed in order when no site-specific config exists. + */ +const NEXT_DATA_CONTENT_PATHS = [ + 'props.pageProps.content.body', + 'props.pageProps.article.body', + 'props.pageProps.article.content', + 'props.pageProps.post.body', + 'props.pageProps.post.content', + 'props.pageProps.data.body', +]; + +/** + * Check if a value looks like an array of content blocks (has objects with type/text fields). + */ +function isContentBlockArray(value: unknown): value is ContentBlock[] { + if (!Array.isArray(value) || value.length === 0) return false; + // Check first few items for content block structure + const sample = value.slice(0, 5); + return sample.some( + (item) => + item && typeof item === 'object' && ('type' in item || 'text' in item || 'textHtml' in item) + ); +} + +/** + * Try to extract metadata for content block arrays from various pageProps locations. + */ +function extractContentBlockMetadata( + pageProps: Record, + document: Document +): { + title: string | null; + byline: string | null; + excerpt: string | null; + publishedTime: string | null; +} { + // Try content.headline, article.headline, post.title, etc. + const content = pageProps.content as Record | undefined; + const article = pageProps.article as Record | undefined; + const post = pageProps.post as Record | undefined; + + const title = + (content?.headline as string) ?? + (article?.headline as string) ?? + (article?.title as string) ?? + (post?.title as string) ?? + (pageProps.title as string) ?? + extractTitle(document); + + const byline = + extractBylineFromAuthors(content?.authors) ?? + extractBylineFromAuthors(article?.authors) ?? + extractBylineFromAuthors(post?.authors) ?? + (pageProps.author as { name?: string })?.name ?? + null; + + const excerpt = + (content?.description as string) ?? + (article?.description as string) ?? + (article?.excerpt as string) ?? + (post?.excerpt as string) ?? + (pageProps.description as string) ?? + null; + + const publishedTime = + (content?.datePublished as string) ?? + (article?.datePublished as string) ?? + (article?.publishedAt as string) ?? + (post?.date as string) ?? + (pageProps.publishedAt as string) ?? + extractPublishedTime(document); + + return { title, byline, excerpt, publishedTime }; +} + +/** + * Try to extract an ExtractionResult from a string body (HTML or plain text). + * Shared by both the custom-path and auto-detect branches of tryNextDataExtraction. + */ +function tryStringBodyExtraction( + content: string, + meta: Record, + document: Document +): ExtractionResult | null { + if (content.length < MIN_CONTENT_LENGTH) return null; + + const isHtml = /<(?:p|div)>/.test(content); + const sanitized = isHtml ? sanitizeHtml(content) : content; + const textContent = isHtml ? extractTextFromHtml(sanitized) : content; + if (textContent.length < MIN_CONTENT_LENGTH) return null; + + return { + title: (meta.headline as string) ?? (meta.title as string) ?? extractTitle(document), + byline: extractBylineFromAuthors(meta.authors) ?? (meta.byline as string) ?? null, + content: isHtml ? sanitized : textContent, + textContent, + excerpt: + (meta.description as string) ?? + (meta.excerpt as string) ?? + generateExcerpt(null, textContent), + siteName: extractSiteName(document), + publishedTime: + (meta.datePublished as string) ?? + (meta.publishedAt as string) ?? + extractPublishedTime(document), + lang: document.documentElement.lang || 'en', + method: isHtml ? 'next-data-html' : 'next-data', + }; +} + +/** + * Strategy 4: Extract from Next.js __NEXT_DATA__ + * Some sites embed full article content in the page props JSON. + * Supports three modes (tried in order): + * 1. Site-specific: Use nextDataPath config to specify custom JSON path + * 2. Auto-detect: Probe common content paths for content block arrays or string bodies + * 3. Default: Look for story.body.content structured blocks + */ +export function tryNextDataExtraction(document: Document, url: string): ExtractionResult | null { + try { + const script = document.querySelector('script#__NEXT_DATA__'); + if (!script?.textContent) return null; + + const data = JSON.parse(script.textContent); + const pageProps = data?.props?.pageProps; + if (!pageProps) return null; + + // Try site-specific path first + const customPath = getSiteNextDataPath(url); + if (customPath) { + const content = getNestedValue(data, customPath); + + // Handle array of content blocks + if (Array.isArray(content)) { + const textContent = extractTextFromContentBlocks( + content.filter((item): item is ContentBlock => item && typeof item === 'object') + ); + if (textContent.length >= MIN_CONTENT_LENGTH) { + logger.debug( + { url, path: customPath }, + 'Next.js custom path (array) extraction succeeded' + ); + // Extract metadata from content structure + const contentMeta = pageProps.content ?? {}; + return { + title: contentMeta.headline ?? pageProps.title ?? extractTitle(document), + byline: extractBylineFromAuthors(contentMeta.authors) ?? pageProps.author?.name ?? null, + content: textContent, + textContent, + excerpt: + contentMeta.description ?? + pageProps.teaser_body ?? + generateExcerpt(null, textContent), + siteName: extractSiteName(document), + publishedTime: + contentMeta.datePublished ?? + pageProps.publishedAt ?? + pageProps.changed_formatted ?? + extractPublishedTime(document), + lang: document.documentElement.lang || 'en', + method: 'next-data', + }; + } + } + + // Handle string content (HTML or plain text) + if (typeof content === 'string') { + // Map pageProps fields to the generic metadata shape expected by tryStringBodyExtraction + const meta: Record = { + title: pageProps.title, + byline: pageProps.author?.name ?? null, + description: pageProps.teaser_body ?? pageProps.description, + publishedAt: pageProps.publishedAt ?? pageProps.changed_formatted, + }; + const result = tryStringBodyExtraction(content, meta, document); + if (result) { + logger.debug({ url, path: customPath }, 'Next.js custom path extraction succeeded'); + return result; + } + } + } + + // Auto-detect: Probe common content paths for content block arrays or string bodies + for (const path of NEXT_DATA_CONTENT_PATHS) { + const content = getNestedValue(data, path); + if (isContentBlockArray(content)) { + const textContent = extractTextFromContentBlocks( + content.filter((item): item is ContentBlock => item && typeof item === 'object') + ); + if (textContent.length >= MIN_CONTENT_LENGTH) { + logger.debug({ url, path }, 'Next.js auto-detected content path extraction succeeded'); + const meta = extractContentBlockMetadata(pageProps as Record, document); + return { + title: meta.title ?? extractTitle(document), + byline: meta.byline, + content: textContent, + textContent, + excerpt: meta.excerpt ?? generateExcerpt(null, textContent), + siteName: extractSiteName(document), + publishedTime: meta.publishedTime ?? extractPublishedTime(document), + lang: document.documentElement.lang || 'en', + method: 'next-data', + }; + } + } + + // Handle string content (HTML or plain text) + if (typeof content === 'string') { + const parentPath = path.split('.').slice(0, -1).join('.'); + const parent = (getNestedValue(data, parentPath) ?? {}) as Record; + // Fall back to pageProps.title when parent has no title fields + const metaWithFallback = { ...parent, title: parent.title ?? pageProps.title }; + const result = tryStringBodyExtraction(content, metaWithFallback, document); + if (result) { + logger.debug({ url, path }, 'Next.js auto-detected string content extraction succeeded'); + return result; + } + } + } + + // Default: Try story.body.content structured blocks + const story = pageProps?.story; + if (!story?.body?.content) return null; + + // Extract text from the structured content + const textParts: string[] = []; + + function extractText(node: unknown): void { + if (!node || typeof node !== 'object') return; + + const obj = node as Record; + + // Handle text nodes + if (obj.type === 'text' && typeof obj.value === 'string') { + textParts.push(obj.value); + return; + } + + // Skip non-content types + const skipTypes = ['inline-newsletter', 'ad', 'related-content', 'inline-recirc']; + if (typeof obj.type === 'string' && skipTypes.includes(obj.type)) { + return; + } + + // Add newlines after paragraphs + if (obj.type === 'paragraph') { + if (Array.isArray(obj.content)) { + obj.content.forEach(extractText); + } + textParts.push('\n\n'); + return; + } + + // Recurse into content arrays + if (Array.isArray(obj.content)) { + obj.content.forEach(extractText); + } + } + + // Process all content blocks + if (Array.isArray(story.body.content)) { + story.body.content.forEach(extractText); + } + + const textContent = textParts.join('').trim(); + if (textContent.length < MIN_CONTENT_LENGTH) return null; + + // Extract author + let byline: string | null = null; + if (Array.isArray(story.authors) && story.authors.length > 0) { + byline = story.authors + .map((a: { name?: string }) => a.name) + .filter(Boolean) + .join(', '); + } + + return { + title: story.headline ?? story.title ?? extractTitle(document), + byline, + content: textContent, + textContent, + excerpt: story.abstract?.[0] ?? generateExcerpt(null, textContent), + siteName: extractSiteName(document), + publishedTime: story.publishedAt ?? extractPublishedTime(document), + lang: document.documentElement.lang || 'en', + method: 'next-data', + }; + } catch (e) { + logger.debug({ url, error: String(e) }, 'Next.js data extraction failed'); + return null; + } +} + +/** + * Extract the Next.js buildId from a __NEXT_DATA__ script tag. + * Returns null if the page is not a Next.js page or has no buildId. + */ +export function extractNextBuildId(document: Document): string | null { + try { + const script = document.querySelector('script#__NEXT_DATA__'); + if (!script?.textContent) return null; + const data = JSON.parse(script.textContent); + const buildId = data?.buildId; + return typeof buildId === 'string' && /^[\w-]+$/.test(buildId) ? buildId : null; + } catch { + return null; + } +} diff --git a/src/extract/readability-extractor.ts b/src/extract/readability-extractor.ts new file mode 100644 index 0000000..9655c75 --- /dev/null +++ b/src/extract/readability-extractor.ts @@ -0,0 +1,70 @@ +/** + * Strategy 1: Extract using Mozilla Readability + */ +import { Readability } from '@mozilla/readability'; + +import { type ExtractionResult, MIN_CONTENT_LENGTH } from './types.js'; +import { + extractPublishedTime, + extractTitle, + extractSiteName, + generateExcerpt, +} from './metadata-extractors.js'; +import { logger } from '../logger.js'; + +/** + * Build an ExtractionResult from a Readability article and the original document. + */ +function buildReadabilityResult( + article: ReturnType['parse']>, + document: Document, + method: string +): ExtractionResult | null { + if (!article?.textContent || article.textContent.length < MIN_CONTENT_LENGTH) { + return null; + } + + return { + title: article.title ?? extractTitle(document), + byline: article.byline ?? null, + content: article.content ?? null, + textContent: article.textContent ?? null, + excerpt: generateExcerpt(article.excerpt ?? null, article.textContent ?? null), + siteName: article.siteName ?? extractSiteName(document), + publishedTime: extractPublishedTime(document) ?? article.publishedTime ?? null, + lang: article.lang ?? null, + method, + }; +} + +/** + * Strategy 1: Extract using Mozilla Readability + * Tries strict mode first, then retries with charThreshold: 100 for unusual DOM structures. + */ +export function tryReadability(document: Document, url: string): ExtractionResult | null { + try { + // Strict pass (default charThreshold of 500) + const clone = document.cloneNode(true) as Document; + const strictResult = buildReadabilityResult( + new Readability(clone).parse(), + document, + 'readability' + ); + if (strictResult) return strictResult; + + // Relaxed pass — lower charThreshold to catch unusual DOM structures + const relaxedClone = document.cloneNode(true) as Document; + const relaxedResult = buildReadabilityResult( + new Readability(relaxedClone, { charThreshold: 100 }).parse(), + document, + 'readability-relaxed' + ); + if (relaxedResult) { + logger.debug({ url }, 'Readability relaxed pass succeeded'); + } + return relaxedResult; + } catch (e) { + logger.debug({ url, error: String(e) }, 'Readability extraction failed'); + return null; + } +} diff --git a/src/extract/rsc-extractor.ts b/src/extract/rsc-extractor.ts new file mode 100644 index 0000000..42a906d --- /dev/null +++ b/src/extract/rsc-extractor.ts @@ -0,0 +1,120 @@ +/** + * Strategy 6: Extract from Next.js RSC (React Server Components) streaming payload. + * App Router pages embed article text inside self.__next_f.push() script calls + * rather than rendering it into the DOM. + */ +import { parseHTML } from 'linkedom'; + +import { type ExtractionResult, MIN_CONTENT_LENGTH } from './types.js'; +import { + extractPublishedTime, + extractTitle, + extractSiteName, + generateExcerpt, +} from './metadata-extractors.js'; +import { logger } from '../logger.js'; + +/** Minimum length for RSC text segments to be considered article content */ +const RSC_MIN_SEGMENT_LENGTH = 100; + +/** + * Check whether a string looks like natural language (not HTML/JS). + */ +function isNaturalLanguage(text: string): boolean { + const len = text.length; + if (len === 0) return false; + + // Reject HTML-heavy content (more than 1 tag per 100 chars) + const tagCount = text.split('<').length - 1; + if (tagCount > len / 100) return false; + + // Reject JS-heavy content + if (text.includes('function(') || text.includes('function (')) return false; + const arrowCount = text.split('=>').length - 1; + if (arrowCount > 3) return false; + + // Must have word-like content: spaces between words + const spaceCount = text.split(' ').length - 1; + return spaceCount / len > 0.1; +} + +export function tryNextRscExtraction(html: string, url: string): ExtractionResult | null { + try { + if (!html.includes('self.__next_f.push(')) return null; + + // Extract all self.__next_f.push([...]) calls, anchored on boundary + const pushPattern = /self\.__next_f\.push\(([\s\S]*?)\)<\/script>/g; + const chunks: string[] = []; + let match; + while ((match = pushPattern.exec(html)) !== null) { + try { + const arr = JSON.parse(match[1]); + if (Array.isArray(arr) && arr[0] === 1 && typeof arr[1] === 'string') { + chunks.push(arr[1]); + } + } catch { + // Skip malformed JSON (some chunks contain unescaped JS) + } + } + + if (chunks.length === 0) return null; + + // Concatenate all type-1 chunks into a single stream + const stream = chunks.join(''); + + // Extract text segments from RSC T markers: id:Thexlen, + const textSegments: string[] = []; + const tMarkerPattern = /[0-9a-f]+:T[0-9a-f]+,/g; + let tMatch; + const tPositions: number[] = []; + while ((tMatch = tMarkerPattern.exec(stream)) !== null) { + tPositions.push(tMatch.index + tMatch[0].length); + } + + // For each T marker, extract text until next RSC row prefix or end + const rowPrefixPattern = /\n[0-9a-f]+:[A-Z["$]/; + for (const pos of tPositions) { + const rest = stream.slice(pos); + const nextRow = rest.search(rowPrefixPattern); + const segment = nextRow === -1 ? rest : rest.slice(0, nextRow); + + if (segment.length >= RSC_MIN_SEGMENT_LENGTH && isNaturalLanguage(segment)) { + textSegments.push(segment.trim()); + } + } + + // Also check for continuation chunks (raw text with no row prefix) + for (const chunk of chunks) { + if ( + chunk.length >= RSC_MIN_SEGMENT_LENGTH && + !/^[0-9a-f]+:/.test(chunk) && + isNaturalLanguage(chunk) + ) { + textSegments.push(chunk.trim()); + } + } + + // Deduplicate (continuation chunks may overlap with T marker extraction) + const unique = [...new Set(textSegments)]; + const textContent = unique.join('\n\n'); + + if (textContent.length < MIN_CONTENT_LENGTH) return null; + + const { document } = parseHTML(html); + + return { + title: extractTitle(document), + byline: null, + content: textContent, + textContent, + excerpt: generateExcerpt(null, textContent), + siteName: extractSiteName(document), + publishedTime: extractPublishedTime(document), + lang: document.documentElement.lang || null, + method: 'next-rsc', + }; + } catch (e) { + logger.debug({ url, error: String(e) }, 'Next.js RSC extraction failed'); + return null; + } +} diff --git a/src/extract/selector-extractor.ts b/src/extract/selector-extractor.ts new file mode 100644 index 0000000..7300cb7 --- /dev/null +++ b/src/extract/selector-extractor.ts @@ -0,0 +1,86 @@ +/** + * Strategy 2: Extract using CSS selectors (like Python implementation) + */ +import { type ExtractionResult, MIN_CONTENT_LENGTH } from './types.js'; +import { + extractPublishedTime, + extractTitle, + extractSiteName, + generateExcerpt, +} from './metadata-extractors.js'; +import { logger } from '../logger.js'; + +// Selectors for finding article content (from Python implementation) +const CONTENT_SELECTORS = [ + 'article', + 'main article', + '[role="main"] article', + '.article-content', + '.article-body', + '.post-content', + '.entry-content', + '.story-body', + '#article-body', + '[itemprop="articleBody"]', + 'main', + '[role="main"]', +]; + +// Elements to remove before extraction +const REMOVE_SELECTORS = [ + 'script', + 'style', + 'nav', + 'aside', + 'footer', + 'header', + 'form', + 'iframe', + '.advertisement', + '.ads', + '.social-share', + '.related-articles', + '.comments', +]; + +/** + * Strategy 2: Extract using CSS selectors (like Python implementation) + */ +export function trySelectorExtraction(document: Document, url: string): ExtractionResult | null { + try { + for (const selector of CONTENT_SELECTORS) { + const el = document.querySelector(selector); + if (!el) continue; + + // Clone and clean the element + const clone = el.cloneNode(true) as Element; + + // Remove unwanted elements + for (const removeSelector of REMOVE_SELECTORS) { + const toRemove = clone.querySelectorAll(removeSelector); + toRemove.forEach((node) => node.remove()); + } + + // Get text content + const textContent = clone.textContent?.trim() ?? ''; + + if (textContent.length >= MIN_CONTENT_LENGTH) { + return { + title: extractTitle(document), + byline: null, + content: clone.innerHTML, + textContent, + excerpt: generateExcerpt(null, textContent), + siteName: extractSiteName(document), + publishedTime: extractPublishedTime(document), + lang: document.documentElement.lang || null, + method: `selector:${selector}`, + }; + } + } + } catch (e) { + logger.debug({ url, error: String(e) }, 'Selector extraction failed'); + } + + return null; +} diff --git a/src/extract/text-density-extractor.ts b/src/extract/text-density-extractor.ts new file mode 100644 index 0000000..f40687f --- /dev/null +++ b/src/extract/text-density-extractor.ts @@ -0,0 +1,44 @@ +/** + * Strategy 5: Extract using text density analysis (CETD algorithm). + * Statistical approach based on text-to-tag density ratios per DOM node. + * Complementary to Readability's heuristic class/id scoring. + */ +import { parseHTML } from 'linkedom'; +import { extractContent } from '@wrtnlabs/web-content-extractor'; + +import { type ExtractionResult, MIN_CONTENT_LENGTH } from './types.js'; +import { + extractPublishedTime, + extractTitle, + extractSiteName, + generateExcerpt, +} from './metadata-extractors.js'; +import { logger } from '../logger.js'; + +export function tryTextDensityExtraction(html: string, url: string): ExtractionResult | null { + try { + const extracted = extractContent(html); + + if (extracted.content.length < MIN_CONTENT_LENGTH) { + return null; + } + + // Parse with linkedom for metadata extraction (reuses the same DOM lib as other strategies) + const { document } = parseHTML(html); + + return { + title: extractTitle(document), + byline: null, + content: extracted.contentHtmls.join(''), + textContent: extracted.content, + excerpt: generateExcerpt(extracted.description ?? null, extracted.content), + siteName: extractSiteName(document), + publishedTime: extractPublishedTime(document), + lang: document.documentElement.lang || null, + method: 'text-density', + }; + } catch (e) { + logger.debug({ url, error: String(e) }, 'Text density extraction failed'); + return null; + } +} diff --git a/src/fetch/http-fetch.ts b/src/fetch/http-fetch.ts index 74ea415..9452e95 100644 --- a/src/fetch/http-fetch.ts +++ b/src/fetch/http-fetch.ts @@ -4,17 +4,11 @@ import httpcloak from 'httpcloak'; import { httpRequest, httpPost, type HttpResponse } from './http-client.js'; import { quickValidate } from './content-validator.js'; -import { - extractFromHtml, - detectWpRestApi, - tryNextDataExtraction, - extractNextBuildId, -} from '../extract/content-extractors.js'; +import { extractFromHtml, tryNextDataExtraction } from '../extract/content-extractors.js'; import { getSiteUserAgent, getSiteReferer, siteUseWpRestApi, - getSiteWpJsonApiPath, siteUseNextData, } from '../sites/site-config.js'; import { @@ -27,19 +21,17 @@ import { fetchRemotePdfBuffer } from './pdf-fetch.js'; import { countWords } from '../extract/utils.js'; import { detectWpAjaxContent, parseWpAjaxResponse } from '../extract/wp-ajax-content.js'; import { logger } from '../logger.js'; -import { htmlToMarkdown } from '../extract/markdown.js'; import type { ExtractionResult, SelectorOptions } from '../extract/types.js'; -import type { FetchResult, ValidationError } from './types.js'; -import { parseHTML } from 'linkedom'; import { GOOD_CONTENT_LENGTH } from '../extract/types.js'; - -/** Shared request context threaded through internal extraction helpers. */ -interface RequestContext { - preset?: string; - timeout?: number; - proxy?: string; - cookies?: Record; -} +import type { FetchResult, RequestContext, ValidationError } from './types.js'; +import { parseHTML } from 'linkedom'; +import { + tryWpRestApiExtraction, + enrichWpMetadata, + extractSlugFromUrl, + resolveWpApiUrl, +} from './wp-rest-api.js'; +import { fetchNextDataRoute } from './next-data-route.js'; // Minimum content length (chars) for successful extraction const MIN_EXTRACTION_LENGTH = 100; @@ -145,303 +137,6 @@ function successResult( }; } -/** Strip HTML tags and decode entities by parsing a fragment and returning its text content. */ -function htmlToText(html: string): string | null { - const { document } = parseHTML(`
${html}
`); - return document.querySelector('div')?.textContent?.trim() ?? null; -} - -/** - * Detect if a WP REST API response is a PMC list (has pmc_list_order in meta). - * Returns the array of list item IDs if found, null otherwise. - */ -function detectPmcListOrder(post: Record): number[] | null { - const meta = post.meta as Record | undefined; - const listOrder = meta?.pmc_list_order; - - if (!Array.isArray(listOrder) || listOrder.length === 0) return null; - if (!listOrder.every((id) => typeof id === 'number')) return null; - - return listOrder as number[]; -} - -/** Maximum list items to fetch to prevent abuse */ -const MAX_LIST_ITEMS = 200; - -/** Batch size for WP REST API requests */ -const WP_LIST_BATCH_SIZE = 50; - -/** - * Fetch WP list items in batch and concatenate their content. - * Returns combined HTML content from all list items. - */ -async function fetchPmcListItems( - origin: string, - itemIds: number[], - ctx: RequestContext -): Promise { - // Limit items to prevent abuse - const limitedIds = itemIds.slice(0, MAX_LIST_ITEMS); - if (itemIds.length > MAX_LIST_ITEMS) { - logger.debug( - { itemCount: itemIds.length, limit: MAX_LIST_ITEMS }, - 'WP list truncated to maximum items' - ); - } - - const allContents: string[] = []; - - for (let i = 0; i < limitedIds.length; i += WP_LIST_BATCH_SIZE) { - const batch = limitedIds.slice(i, i + WP_LIST_BATCH_SIZE); - const includeParam = batch.join(','); - const apiUrl = `${origin}/wp-json/wp/v2/pmc_list_item?include=${includeParam}&per_page=${WP_LIST_BATCH_SIZE}`; - - logger.debug({ apiUrl, count: batch.length }, 'Fetching WP list items batch'); - - const response = await httpRequest( - apiUrl, - { Accept: 'application/json' }, - ctx.preset, - ctx.timeout, - ctx.proxy, - ctx.cookies - ); - if (!response.success || !response.html) continue; - - let items: unknown[]; - try { - const parsed = JSON.parse(response.html); - if (!Array.isArray(parsed)) continue; - items = parsed; - } catch { - logger.debug({ apiUrl }, 'Failed to parse WP list items response'); - continue; - } - - // Create a map for ordering - const itemMap = new Map(); - for (const item of items) { - if (typeof item !== 'object' || !item) continue; - const id = (item as Record).id; - if (typeof id !== 'number') continue; - const contentHtml = resolveWpField((item as Record).content); - if (contentHtml) { - itemMap.set(id, contentHtml); - } - } - - // Add in original order - for (const id of batch) { - const content = itemMap.get(id); - if (content) { - allContents.push(content); - } - } - } - - if (allContents.length === 0) return null; - - return allContents.join('\n\n'); -} - -/** - * Resolve a WP REST API field that may be a plain string (custom endpoints) - * or an object with a `rendered` property (standard WP). - */ -function resolveWpField(field: unknown): string | undefined { - if (typeof field === 'string') return field; - if (typeof field === 'object' && field !== null && 'rendered' in field) { - const rendered = (field as { rendered: unknown }).rendered; - if (typeof rendered === 'string') return rendered; - } - return undefined; -} - -/** - * Extract the first post object from a WP REST API response. - * - * Handles three response shapes: - * - Standard ?slug= query: [{...}] - * - Standard /posts/123: {...} - * - Custom envelope: {posts: [{...}]} - */ -function resolveWpPost(raw: unknown): Record | null { - if (Array.isArray(raw)) { - const post = raw[0]; - return typeof post === 'object' && post !== null ? (post as Record) : null; - } - if (typeof raw === 'object' && raw !== null) { - const obj = raw as Record; - if (Array.isArray(obj.posts)) { - const post = obj.posts[0]; - return typeof post === 'object' && post !== null ? (post as Record) : null; - } - return obj; - } - return null; -} - -/** - * Check if WP REST API content contains a teaser link added by WordPress - * when content is split at the tag. WordPress appends a tracking - * parameter (utm_campaign=api) to the "read more" link in API responses. - */ -function hasWpApiTruncationMarker(html: string): boolean { - return /href="[^"]*utm_campaign=api/.test(html); -} - -/** - * Try fetching article content from WordPress REST API. - * Appends ?_embed to the URL to get author data in the response. - * Detects PMC lists and fetches all list items for complete content. - * Returns an ExtractionResult if API returns sufficient content, null otherwise. - */ -async function tryWpRestApiExtraction( - apiUrl: string, - originalResult: ExtractionResult | null, - ctx: RequestContext -): Promise { - try { - const embedUrl = apiUrl + (apiUrl.includes('?') ? '&_embed' : '?_embed'); - logger.info({ apiUrl: embedUrl }, 'Trying WordPress REST API extraction'); - - const response = await httpRequest( - embedUrl, - { Accept: 'application/json' }, - ctx.preset, - ctx.timeout, - ctx.proxy, - ctx.cookies - ); - - if (!response.success || !response.html) return null; - - const json = resolveWpPost(JSON.parse(response.html)); - if (!json) return null; - - // Check for PMC list structure (has pmc_list_order in meta) - const pmcListOrder = detectPmcListOrder(json); - let contentHtml = resolveWpField(json.content) ?? ''; - let method = 'wp-rest-api'; - - if (pmcListOrder && pmcListOrder.length > 0) { - // This is a PMC list - fetch all list items - const origin = new URL(apiUrl).origin; - logger.info( - { apiUrl, itemCount: pmcListOrder.length }, - 'Detected PMC list, fetching list items' - ); - - const listItemsHtml = await fetchPmcListItems(origin, pmcListOrder, ctx); - if (listItemsHtml) { - // Combine intro content with list items, avoiding leading whitespace - contentHtml = contentHtml ? contentHtml + '\n\n' + listItemsHtml : listItemsHtml; - method = 'wp-rest-api-pmc-list'; - } - } - - if (!contentHtml) return null; - - // Reject content that contains a WordPress API teaser link, indicating the - // full article was not included in the API response. - if (hasWpApiTruncationMarker(contentHtml)) { - logger.debug({ apiUrl }, 'WP API returned teaser content, skipping'); - return null; - } - - const textContent = htmlToText(contentHtml) ?? ''; - if (textContent.length < GOOD_CONTENT_LENGTH) return null; - - const rawTitle = resolveWpField(json.title); - const title = rawTitle ? (htmlToText(rawTitle) ?? null) : (originalResult?.title ?? null); - - const rawExcerpt = resolveWpField(json.excerpt); - const excerpt = rawExcerpt ? (htmlToText(rawExcerpt) ?? null) : null; - - const embedded = json._embedded as { author?: { name?: string }[] } | undefined; - const byline = embedded?.author?.[0]?.name ?? originalResult?.byline ?? null; - const dateGmt = typeof json.date_gmt === 'string' ? json.date_gmt : null; - - return { - title, - byline, - content: contentHtml, - textContent, - excerpt, - siteName: originalResult?.siteName ?? null, - publishedTime: dateGmt ?? originalResult?.publishedTime ?? null, - lang: originalResult?.lang ?? null, - markdown: htmlToMarkdown(contentHtml), - method, - }; - } catch (e) { - logger.debug({ apiUrl, error: String(e) }, 'WordPress REST API extraction failed'); - return null; - } -} - -/** - * Enrich a DOM extraction result with structured metadata from a WP REST API response. - * API metadata (title, byline, date, excerpt) is preferred when available since - * the REST API provides well-structured fields independent of page layout. - */ -function enrichWpMetadata( - domResult: ExtractionResult, - wpResult: ExtractionResult -): ExtractionResult { - return { - ...domResult, - title: wpResult.title ?? domResult.title, - byline: wpResult.byline ?? domResult.byline, - excerpt: wpResult.excerpt ?? domResult.excerpt, - publishedTime: wpResult.publishedTime ?? domResult.publishedTime, - markdown: wpResult.markdown ?? domResult.markdown, - }; -} - -/** - * Extract the last non-empty path segment from a URL to use as a post slug. - */ -function extractSlugFromUrl(url: string): string | null { - try { - const pathname = new URL(url).pathname; - const segments = pathname.split('/').filter(Boolean); - return segments.length > 0 ? segments[segments.length - 1] : null; - } catch { - return null; - } -} - -/** - * Resolve WP REST API URL using auto-detection from HTML, then config-driven construction. - * Returns the API endpoint URL or null if no WP API is available. - * - * Order of operations: - * 1. Auto-detect WP API URL from HTML () - * 2. If not found + site has wpJsonApiPath -> construct custom API URL from slug - * 3. If not found + site has useWpRestApi -> construct standard API URL from slug - */ -function resolveWpApiUrl(html: string, url: string): string | null { - // 1. Auto-detect from HTML - const detected = detectWpRestApi(parseHTML(html).document, url); - if (detected) return detected; - - // 2. Config-driven: custom API path - const customPath = getSiteWpJsonApiPath(url); - if (customPath) { - const slug = extractSlugFromUrl(url); - if (slug) return `${new URL(url).origin}${customPath}${encodeURIComponent(slug)}`; - } - - // 3. Config-driven: standard WP REST API - if (siteUseWpRestApi(url)) { - const slug = extractSlugFromUrl(url); - if (slug) return `${new URL(url).origin}/wp-json/wp/v2/posts?slug=${encodeURIComponent(slug)}`; - } - - return null; -} - /** * Try WP REST API fallback and wrap the result into a FetchResult. * Returns a success FetchResult if API returns sufficient content, null otherwise. @@ -498,17 +193,6 @@ function tryNextDataFallback( } } -/** - * Construct the Next.js data route URL from a buildId and page URL. - * Example: buildId="abc", url="https://example.com/section/slug" - * => "https://example.com/_next/data/abc/section/slug.json" - */ -function buildNextDataRouteUrl(url: string, buildId: string): string { - const parsed = new URL(url); - const pathname = parsed.pathname.replace(/\/$/, '') || '/index'; - return `${parsed.origin}/_next/data/${buildId}${pathname}.json`; -} - /** * Try fetching full content from the Next.js /_next/data/ route. * This route sometimes returns richer content than the SSR HTML. @@ -522,58 +206,23 @@ async function tryNextDataRoute( domExtracted: ExtractionResult | null, ctx: RequestContext ): Promise { - const { document } = parseHTML(html); - const buildId = extractNextBuildId(document); - if (!buildId) return null; - - const dataRouteUrl = buildNextDataRouteUrl(url, buildId); - - try { - logger.debug({ url, dataRouteUrl }, 'Trying Next.js data route'); - - const dataResponse = await httpRequest( - dataRouteUrl, - { Accept: 'application/json' }, - ctx.preset, - ctx.timeout, - ctx.proxy, - ctx.cookies - ); - if (!dataResponse.success || !dataResponse.html) return null; - - const json = JSON.parse(dataResponse.html); - const pageProps = json.pageProps; - if (!pageProps) return null; + const result = await fetchNextDataRoute(html, url, ctx); + if (!result || !result.textContent) return null; - // Build a synthetic __NEXT_DATA__ document so tryNextDataExtraction can process it - const syntheticData = JSON.stringify({ buildId, props: { pageProps } }).replace( - /`; - const { document: syntheticDoc } = parseHTML(syntheticHtml); - - const result = tryNextDataExtraction(syntheticDoc, url); - if (!result || !result.textContent) return null; - - // Only use data route result if it has more content than DOM extraction - const domLen = domExtracted?.textContent?.length ?? 0; - if (result.textContent.length <= domLen) return null; + // Only use data route result if it has more content than DOM extraction + const domLen = domExtracted?.textContent?.length ?? 0; + if (result.textContent.length <= domLen) return null; - logger.info( - { url, dataRouteUrl, dataLen: result.textContent.length, domLen }, - 'Next.js data route returned more content' - ); + logger.info( + { url, dataLen: result.textContent.length, domLen }, + 'Next.js data route returned more content' + ); - return successResult(url, startTime, result, { - statusCode: response.statusCode, - rawHtml: process.env.RECORD_HTML === 'true' ? (response.html ?? null) : null, - extractionMethod: 'next-data-route', - }); - } catch (e) { - logger.debug({ url, dataRouteUrl, error: String(e) }, 'Next.js data route failed'); - return null; - } + return successResult(url, startTime, result, { + statusCode: response.statusCode, + rawHtml: process.env.RECORD_HTML === 'true' ? (response.html ?? null) : null, + extractionMethod: 'next-data-route', + }); } /** @@ -870,6 +519,12 @@ export async function httpFetch(url: string, options: HttpFetchOptions = {}): Pr return extractPdfFromBuffer(buffer, url, response.statusCode); } + /** Attach raw HTML to a fallback FetchResult when recording is enabled. */ + function attachRawHtml(result: FetchResult): FetchResult { + if (keepRawHtml) result.rawHtml = response.html; + return result; + } + // Validate content const validation = quickValidate( response.html, @@ -881,10 +536,7 @@ export async function httpFetch(url: string, options: HttpFetchOptions = {}): Pr // Try Next.js data extraction and WP REST API for insufficient content if (validation.error === 'insufficient_content') { const nextDataFallback = tryNextDataFallback(response.html, url, startTime, response); - if (nextDataFallback) { - if (keepRawHtml) nextDataFallback.rawHtml = response.html; - return nextDataFallback; - } + if (nextDataFallback) return attachRawHtml(nextDataFallback); const wpFallback = await tryWpRestApiFallback( response.html, @@ -894,10 +546,7 @@ export async function httpFetch(url: string, options: HttpFetchOptions = {}): Pr null, ctx ); - if (wpFallback) { - if (keepRawHtml) wpFallback.rawHtml = response.html; - return wpFallback; - } + if (wpFallback) return attachRawHtml(wpFallback); const prismFallback = await tryPrismContentApiFallback( response.html, @@ -906,10 +555,7 @@ export async function httpFetch(url: string, options: HttpFetchOptions = {}): Pr response, ctx ); - if (prismFallback) { - if (keepRawHtml) prismFallback.rawHtml = response.html; - return prismFallback; - } + if (prismFallback) return attachRawHtml(prismFallback); const wpAjaxFallback = await tryWpAjaxContentFallback( response.html, @@ -919,10 +565,7 @@ export async function httpFetch(url: string, options: HttpFetchOptions = {}): Pr null, ctx ); - if (wpAjaxFallback) { - if (keepRawHtml) wpAjaxFallback.rawHtml = response.html; - return wpAjaxFallback; - } + if (wpAjaxFallback) return attachRawHtml(wpAjaxFallback); } return failResult( @@ -983,10 +626,7 @@ export async function httpFetch(url: string, options: HttpFetchOptions = {}): Pr response, ctx ); - if (prismResult) { - if (keepRawHtml) prismResult.rawHtml = response.html; - return prismResult; - } + if (prismResult) return attachRawHtml(prismResult); // Extract content using DOM-based strategies const extracted = extractFromHtml(response.html, url, selectorOpts); @@ -1016,10 +656,7 @@ export async function httpFetch(url: string, options: HttpFetchOptions = {}): Pr extracted, ctx ); - if (wpAjaxResult) { - if (keepRawHtml) wpAjaxResult.rawHtml = response.html; - return wpAjaxResult; - } + if (wpAjaxResult) return attachRawHtml(wpAjaxResult); const wordCount = extracted.textContent ? extracted.textContent.split(/\s+/).length : 0; return failResult( @@ -1045,10 +682,7 @@ export async function httpFetch(url: string, options: HttpFetchOptions = {}): Pr extracted, ctx ); - if (dataRouteResult) { - if (keepRawHtml) dataRouteResult.rawHtml = response.html; - return dataRouteResult; - } + if (dataRouteResult) return attachRawHtml(dataRouteResult); } const latencyMs = Date.now() - startTime; diff --git a/src/fetch/next-data-route.ts b/src/fetch/next-data-route.ts new file mode 100644 index 0000000..691be68 --- /dev/null +++ b/src/fetch/next-data-route.ts @@ -0,0 +1,74 @@ +/** + * Next.js data route extraction. + * + * Fetches content from the Next.js /_next/data/ route, which sometimes + * returns richer content than the SSR HTML. + */ +import { httpRequest } from './http-client.js'; +import { parseHTML } from 'linkedom'; +import { tryNextDataExtraction, extractNextBuildId } from '../extract/content-extractors.js'; +import type { ExtractionResult } from '../extract/types.js'; +import type { RequestContext } from './types.js'; +import { logger } from '../logger.js'; + +/** + * Construct the Next.js data route URL from a buildId and page URL. + * Example: buildId="abc", url="https://example.com/section/slug" + * => "https://example.com/_next/data/abc/section/slug.json" + */ +export function buildNextDataRouteUrl(url: string, buildId: string): string { + const parsed = new URL(url); + const pathname = parsed.pathname.replace(/\/$/, '') || '/index'; + return `${parsed.origin}/_next/data/${buildId}${pathname}.json`; +} + +/** + * Fetch content from the Next.js /_next/data/ route and extract it. + * Returns an ExtractionResult if the data route yields content, null otherwise. + * + * The caller is responsible for comparing against DOM extraction length + * and wrapping the result in a FetchResult. + */ +export async function fetchNextDataRoute( + html: string, + url: string, + ctx: RequestContext +): Promise { + const { document } = parseHTML(html); + const buildId = extractNextBuildId(document); + if (!buildId) return null; + + const dataRouteUrl = buildNextDataRouteUrl(url, buildId); + + try { + logger.debug({ url, dataRouteUrl }, 'Trying Next.js data route'); + + const dataResponse = await httpRequest( + dataRouteUrl, + { Accept: 'application/json' }, + ctx.preset, + ctx.timeout, + ctx.proxy, + ctx.cookies + ); + if (!dataResponse.success || !dataResponse.html) return null; + + const json = JSON.parse(dataResponse.html); + const pageProps = json.pageProps; + if (!pageProps) return null; + + // Build a synthetic __NEXT_DATA__ document so tryNextDataExtraction can process it + const syntheticData = JSON.stringify({ buildId, props: { pageProps } }).replace( + /`; + const { document: syntheticDoc } = parseHTML(syntheticHtml); + + const result = tryNextDataExtraction(syntheticDoc, url); + return result?.textContent ? result : null; + } catch (e) { + logger.debug({ url, dataRouteUrl, error: String(e) }, 'Next.js data route failed'); + return null; + } +} diff --git a/src/fetch/types.ts b/src/fetch/types.ts index c22faac..f319fa2 100644 --- a/src/fetch/types.ts +++ b/src/fetch/types.ts @@ -28,6 +28,14 @@ export interface ValidationResult { }; } +/** Shared request context threaded through internal extraction helpers. */ +export interface RequestContext { + preset?: string; + timeout?: number; + proxy?: string; + cookies?: Record; +} + export interface FetchResult { success: boolean; url: string; diff --git a/src/fetch/wp-rest-api.ts b/src/fetch/wp-rest-api.ts new file mode 100644 index 0000000..ae82652 --- /dev/null +++ b/src/fetch/wp-rest-api.ts @@ -0,0 +1,308 @@ +/** + * WordPress REST API extraction helpers. + * + * Detects WP REST API endpoints, fetches structured post data, and converts + * it into ExtractionResult objects. Supports standard WP endpoints, custom + * API paths, and PMC list articles. + */ +import { httpRequest } from './http-client.js'; +import { parseHTML } from 'linkedom'; +import { detectWpRestApi } from '../extract/content-extractors.js'; +import { getSiteWpJsonApiPath, siteUseWpRestApi } from '../sites/site-config.js'; +import { htmlToMarkdown } from '../extract/markdown.js'; +import { htmlToText } from '../extract/utils.js'; +import type { ExtractionResult } from '../extract/types.js'; +import { GOOD_CONTENT_LENGTH } from '../extract/types.js'; +import type { RequestContext } from './types.js'; +import { logger } from '../logger.js'; + +/** + * Detect if a WP REST API response is a PMC list (has pmc_list_order in meta). + * Returns the array of list item IDs if found, null otherwise. + */ +function detectPmcListOrder(post: Record): number[] | null { + const meta = post.meta as Record | undefined; + const listOrder = meta?.pmc_list_order; + + if (!Array.isArray(listOrder) || listOrder.length === 0) return null; + if (!listOrder.every((id) => typeof id === 'number')) return null; + + return listOrder as number[]; +} + +/** Maximum list items to fetch to prevent abuse */ +const MAX_LIST_ITEMS = 200; + +/** Batch size for WP REST API requests */ +const WP_LIST_BATCH_SIZE = 50; + +/** + * Fetch WP list items in batch and concatenate their content. + * Returns combined HTML content from all list items. + */ +async function fetchPmcListItems( + origin: string, + itemIds: number[], + ctx: RequestContext +): Promise { + // Limit items to prevent abuse + const limitedIds = itemIds.slice(0, MAX_LIST_ITEMS); + if (itemIds.length > MAX_LIST_ITEMS) { + logger.debug( + { itemCount: itemIds.length, limit: MAX_LIST_ITEMS }, + 'WP list truncated to maximum items' + ); + } + + const allContents: string[] = []; + + for (let i = 0; i < limitedIds.length; i += WP_LIST_BATCH_SIZE) { + const batch = limitedIds.slice(i, i + WP_LIST_BATCH_SIZE); + const includeParam = batch.join(','); + const apiUrl = `${origin}/wp-json/wp/v2/pmc_list_item?include=${includeParam}&per_page=${WP_LIST_BATCH_SIZE}`; + + logger.debug({ apiUrl, count: batch.length }, 'Fetching WP list items batch'); + + const response = await httpRequest( + apiUrl, + { Accept: 'application/json' }, + ctx.preset, + ctx.timeout, + ctx.proxy, + ctx.cookies + ); + if (!response.success || !response.html) continue; + + let items: unknown[]; + try { + const parsed = JSON.parse(response.html); + if (!Array.isArray(parsed)) continue; + items = parsed; + } catch { + logger.debug({ apiUrl }, 'Failed to parse WP list items response'); + continue; + } + + // Create a map for ordering + const itemMap = new Map(); + for (const item of items) { + if (typeof item !== 'object' || !item) continue; + const id = (item as Record).id; + if (typeof id !== 'number') continue; + const contentHtml = resolveWpField((item as Record).content); + if (contentHtml) { + itemMap.set(id, contentHtml); + } + } + + // Add in original order + for (const id of batch) { + const content = itemMap.get(id); + if (content) { + allContents.push(content); + } + } + } + + if (allContents.length === 0) return null; + + return allContents.join('\n\n'); +} + +/** + * Resolve a WP REST API field that may be a plain string (custom endpoints) + * or an object with a `rendered` property (standard WP). + */ +function resolveWpField(field: unknown): string | undefined { + if (typeof field === 'string') return field; + if (typeof field === 'object' && field !== null && 'rendered' in field) { + const rendered = (field as { rendered: unknown }).rendered; + if (typeof rendered === 'string') return rendered; + } + return undefined; +} + +/** + * Extract the first post object from a WP REST API response. + * + * Handles three response shapes: + * - Standard ?slug= query: [{...}] + * - Standard /posts/123: {...} + * - Custom envelope: {posts: [{...}]} + */ +function resolveWpPost(raw: unknown): Record | null { + if (Array.isArray(raw)) { + const post = raw[0]; + return typeof post === 'object' && post !== null ? (post as Record) : null; + } + if (typeof raw === 'object' && raw !== null) { + const obj = raw as Record; + if (Array.isArray(obj.posts)) { + const post = obj.posts[0]; + return typeof post === 'object' && post !== null ? (post as Record) : null; + } + return obj; + } + return null; +} + +/** + * Check if WP REST API content contains a teaser link added by WordPress + * when content is split at the tag. WordPress appends a tracking + * parameter (utm_campaign=api) to the "read more" link in API responses. + */ +function hasWpApiTruncationMarker(html: string): boolean { + return /href="[^"]*utm_campaign=api/.test(html); +} + +/** + * Try fetching article content from WordPress REST API. + * Appends ?_embed to the URL to get author data in the response. + * Detects PMC lists and fetches all list items for complete content. + * Returns an ExtractionResult if API returns sufficient content, null otherwise. + */ +export async function tryWpRestApiExtraction( + apiUrl: string, + originalResult: ExtractionResult | null, + ctx: RequestContext +): Promise { + try { + const embedUrl = apiUrl + (apiUrl.includes('?') ? '&_embed' : '?_embed'); + logger.info({ apiUrl: embedUrl }, 'Trying WordPress REST API extraction'); + + const response = await httpRequest( + embedUrl, + { Accept: 'application/json' }, + ctx.preset, + ctx.timeout, + ctx.proxy, + ctx.cookies + ); + + if (!response.success || !response.html) return null; + + const json = resolveWpPost(JSON.parse(response.html)); + if (!json) return null; + + // Check for PMC list structure (has pmc_list_order in meta) + const pmcListOrder = detectPmcListOrder(json); + let contentHtml = resolveWpField(json.content) ?? ''; + let method = 'wp-rest-api'; + + if (pmcListOrder && pmcListOrder.length > 0) { + // This is a PMC list - fetch all list items + const origin = new URL(apiUrl).origin; + logger.info( + { apiUrl, itemCount: pmcListOrder.length }, + 'Detected PMC list, fetching list items' + ); + + const listItemsHtml = await fetchPmcListItems(origin, pmcListOrder, ctx); + if (listItemsHtml) { + // Combine intro content with list items, avoiding leading whitespace + contentHtml = contentHtml ? contentHtml + '\n\n' + listItemsHtml : listItemsHtml; + method = 'wp-rest-api-pmc-list'; + } + } + + if (!contentHtml) return null; + + // Reject content that contains a WordPress API teaser link, indicating the + // full article was not included in the API response. + if (hasWpApiTruncationMarker(contentHtml)) { + logger.debug({ apiUrl }, 'WP API returned teaser content, skipping'); + return null; + } + + const textContent = htmlToText(contentHtml); + if (textContent.length < GOOD_CONTENT_LENGTH) return null; + + const rawTitle = resolveWpField(json.title); + const title = rawTitle ? htmlToText(rawTitle) || null : (originalResult?.title ?? null); + + const rawExcerpt = resolveWpField(json.excerpt); + const excerpt = rawExcerpt ? htmlToText(rawExcerpt) || null : null; + + const embedded = json._embedded as { author?: { name?: string }[] } | undefined; + const byline = embedded?.author?.[0]?.name ?? originalResult?.byline ?? null; + const dateGmt = typeof json.date_gmt === 'string' ? json.date_gmt : null; + + return { + title, + byline, + content: contentHtml, + textContent, + excerpt, + siteName: originalResult?.siteName ?? null, + publishedTime: dateGmt ?? originalResult?.publishedTime ?? null, + lang: originalResult?.lang ?? null, + markdown: htmlToMarkdown(contentHtml), + method, + }; + } catch (e) { + logger.debug({ apiUrl, error: String(e) }, 'WordPress REST API extraction failed'); + return null; + } +} + +/** + * Enrich a DOM extraction result with structured metadata from a WP REST API response. + * API metadata (title, byline, date, excerpt) is preferred when available since + * the REST API provides well-structured fields independent of page layout. + */ +export function enrichWpMetadata( + domResult: ExtractionResult, + wpResult: ExtractionResult +): ExtractionResult { + return { + ...domResult, + title: wpResult.title ?? domResult.title, + byline: wpResult.byline ?? domResult.byline, + excerpt: wpResult.excerpt ?? domResult.excerpt, + publishedTime: wpResult.publishedTime ?? domResult.publishedTime, + markdown: wpResult.markdown ?? domResult.markdown, + }; +} + +/** + * Extract the last non-empty path segment from a URL to use as a post slug. + */ +export function extractSlugFromUrl(url: string): string | null { + try { + const pathname = new URL(url).pathname; + const segments = pathname.split('/').filter(Boolean); + return segments.length > 0 ? segments[segments.length - 1] : null; + } catch { + return null; + } +} + +/** + * Resolve WP REST API URL using auto-detection from HTML, then config-driven construction. + * Returns the API endpoint URL or null if no WP API is available. + * + * Order of operations: + * 1. Auto-detect WP API URL from HTML () + * 2. If not found + site has wpJsonApiPath -> construct custom API URL from slug + * 3. If not found + site has useWpRestApi -> construct standard API URL from slug + */ +export function resolveWpApiUrl(html: string, url: string): string | null { + // 1. Auto-detect from HTML + const detected = detectWpRestApi(parseHTML(html).document, url); + if (detected) return detected; + + // 2. Config-driven: custom API path + const customPath = getSiteWpJsonApiPath(url); + if (customPath) { + const slug = extractSlugFromUrl(url); + if (slug) return `${new URL(url).origin}${customPath}${encodeURIComponent(slug)}`; + } + + // 3. Config-driven: standard WP REST API + if (siteUseWpRestApi(url)) { + const slug = extractSlugFromUrl(url); + if (slug) return `${new URL(url).origin}/wp-json/wp/v2/posts?slug=${encodeURIComponent(slug)}`; + } + + return null; +} From 79c81b471f8c95e6ad4ce6deeaee0031002d8130 Mon Sep 17 00:00:00 2001 From: Teng Lin Date: Sun, 8 Feb 2026 12:41:11 -0500 Subject: [PATCH 2/2] fix: sanitize HTML in JSON-LD articleBody extraction JSON-LD articleBody/text fields can contain raw HTML with script tags, event handlers, and dangerous URI schemes. Apply sanitizeHtml to strip dangerous elements and htmlToText to produce clean textContent, matching the pattern used by next-data-extractor. Co-Authored-By: Claude Opus 4.6 --- src/extract/json-ld-extractor.ts | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/src/extract/json-ld-extractor.ts b/src/extract/json-ld-extractor.ts index edd0b16..b72f304 100644 --- a/src/extract/json-ld-extractor.ts +++ b/src/extract/json-ld-extractor.ts @@ -3,6 +3,7 @@ */ import { type ExtractionResult, MIN_CONTENT_LENGTH } from './types.js'; import { extractPublishedTime, extractSiteName } from './metadata-extractors.js'; +import { sanitizeHtml, htmlToText } from './utils.js'; import { logger } from '../logger.js'; /** JSON-LD article types that contain extractable content */ @@ -63,15 +64,18 @@ function parseJsonLdItem( ): Omit | null { if (!isArticleType(item)) return null; - const content = - (item.articleBody as string) ?? (item.text as string) ?? (item.description as string); - if (!content) return null; + const raw = (item.articleBody as string) ?? (item.text as string) ?? (item.description as string); + if (!raw) return null; + + const isHtml = /<(?:p|div|br|span|h[1-6])[\s>/]/i.test(raw); + const content = isHtml ? sanitizeHtml(raw) : raw; + const textContent = isHtml ? htmlToText(content) : raw; return { title: (item.headline as string) ?? (item.name as string) ?? null, byline: extractAuthorFromJsonLd(item.author), content, - textContent: content, + textContent, excerpt: (item.description as string) ?? null, }; }