Merge pull request #5 from RATCHAW/RATCHAW/fix-hover-bbox-accuracy

Playground: snap hover bboxes to the PDF text layer + show partial extractions

Author: RATCHAW

README.md

@@ -1,5 +1,7 @@
 # extractkit
 
+![extractkit playground — drop in a document, watch fields stream in, hover a field to highlight its source region on the page](./docs/demo.gif)
+
 **Extraction you can audit.** Define a Zod schema, feed it a PDF or image, get back schema-validated JSON where every field carries provenance — the page and bounding box it came from — plus a confidence score.
 
 > **Status: in development.** The core library ([`packages/core`](./packages/core)), the eval harness ([`packages/evals`](./packages/evals)), and the playground ([`apps/playground`](./apps/playground)) are implemented and tested. The first live provider run is in — the OpenAI lineup across the CORD-v2 receipt set — validating core's live path and publishing the [benchmark](#benchmark) below. Still open: the Anthropic and Gemini lineups, the DocILE invoice half (blocked on a dataset token), and the demo GIF. See [ROADMAP.md](./ROADMAP.md).

apps/playground/src/client/App.tsx

@@ -104,14 +104,15 @@ export function App() {
     }
   }, [file, modelId, schemaId]);
 
-  const entries = useMemo<FieldEntry[]>(
-    () => (run.result !== null ? fieldEntries(run.result.fields) : run.live),
-    [run.result, run.live],
-  );
-  const boxes = useMemo(
-    () => entries.filter((e) => e.field.bbox !== null && e.field.page !== null),
-    [entries],
-  );
+  const entries = useMemo<FieldEntry[]>(() => {
+    if (run.result !== null) return fieldEntries(run.result.fields);
+    // A failed run may still carry a partial extraction worth showing.
+    if (run.error?.partial !== undefined) return fieldEntries(run.error.partial.fields);
+    return run.live;
+  }, [run.result, run.error, run.live]);
+  // Include fields without model provenance: the PDF viewer can still locate
+  // their values in the page text layer. Value-less fields have nothing to find.
+  const boxes = useMemo(() => entries.filter((e) => e.field.value !== null), [entries]);
 
   return (
     <div className="app">

apps/playground/src/client/components/DocumentViewer.tsx

@@ -1,11 +1,14 @@
-import { useEffect, useRef, useState } from 'react';
+import { useEffect, useMemo, useRef, useState } from 'react';
 import type { DragEvent } from 'react';
-import { GlobalWorkerOptions, getDocument } from 'pdfjs-dist';
+import { GlobalWorkerOptions, Util, getDocument } from 'pdfjs-dist';
 import type { PDFDocumentLoadingTask, PDFPageProxy } from 'pdfjs-dist';
 import workerSrc from 'pdfjs-dist/build/pdf.worker.min.mjs?url';
+import type { BBox as BBoxRect } from 'extractkit';
 import type { FieldEntry } from '../lib/fields';
 import { formatPath, formatValue } from '../lib/fields';
 import { bboxToStyle } from '../lib/geometry';
+import type { TextSpan } from '../lib/snap';
+import { locateField } from '../lib/snap';
 import { isPdf, pickFile } from '../lib/upload';
 
 GlobalWorkerOptions.workerSrc = workerSrc;
@@ -44,40 +47,79 @@ export function DocumentViewer(props: ViewerProps) {
       {isPdf(file) ? (
         <PdfView file={file} boxes={boxes} activeKey={activeKey} onActivate={onActivate} />
       ) : (
-        <div className="page-stack">
-          <ImagePage
-            src={docUrl}
-            pageIndex={0}
-            boxes={boxes}
-            activeKey={activeKey}
-            onActivate={onActivate}
-          />
-        </div>
+        <ImagePage src={docUrl} boxes={boxes} activeKey={activeKey} onActivate={onActivate} />
       )}
     </div>
   );
 }
 
-interface PageBoxProps {
+/** A field placed on a specific page, with the bbox to draw. */
+interface PlacedBox {
+  entry: FieldEntry;
+  bbox: BBoxRect;
+}
+
+interface OverlayProps {
   boxes: FieldEntry[];
-  pageIndex: number;
   activeKey: string | null;
   onActivate: (key: string | null) => void;
 }
 
-function ImagePage({ src, ...page }: { src: string } & PageBoxProps) {
+function ImagePage({ src, boxes, activeKey, onActivate }: { src: string } & OverlayProps) {
+  // Images have no text layer to snap to; draw the model's own provenance.
+  const placed = useMemo(
+    () =>
+      boxes
+        .filter((box) => box.field.bbox !== null && (box.field.page ?? 0) === 0)
+        .map((box) => ({ entry: box, bbox: box.field.bbox! })),
+    [boxes],
+  );
   return (
-    <div className="page">
-      <img className="page-image" src={src} alt="Uploaded document" />
-      <BoxLayer {...page} />
+    <div className="page-stack">
+      <div className="page">
+        <img className="page-image" src={src} alt="Uploaded document" />
+        <BoxLayer placed={placed} activeKey={activeKey} onActivate={onActivate} />
+      </div>
     </div>
   );
 }
 
-function PdfView({ file, boxes, activeKey, onActivate }: { file: File } & Omit<PageBoxProps, 'pageIndex'>) {
-  const [pages, setPages] = useState<PDFPageProxy[]>([]);
+/** Positioned text runs of a page, normalized 0–1 with a top-left origin. */
+async function pageTextSpans(page: PDFPageProxy): Promise<TextSpan[]> {
+  const viewport = page.getViewport({ scale: 1 });
+  const content = await page.getTextContent();
+  const spans: TextSpan[] = [];
+  for (const item of content.items) {
+    if (!('str' in item) || item.str.trim() === '') continue;
+    const tx = Util.transform(viewport.transform, item.transform);
+    const fontHeight = Math.hypot(tx[2], tx[3]);
+    spans.push({
+      text: item.str,
+      x0: tx[4] / viewport.width,
+      y0: (tx[5] - fontHeight) / viewport.height,
+      x1: (tx[4] + item.width * viewport.scale) / viewport.width,
+      y1: tx[5] / viewport.height,
+    });
+  }
+  return spans;
+}
+
+function PdfView({ file, boxes, activeKey, onActivate }: { file: File } & OverlayProps) {
+  const [pages, setPages] = useState<{ page: PDFPageProxy; spans: TextSpan[] }[]>([]);
   const [error, setError] = useState<string | null>(null);
 
+  // Place every field: snap to the page text layer, rescuing fields whose
+  // model-reported bbox or page is missing or wrong.
+  const placedByPage = useMemo<PlacedBox[][]>(() => {
+    const spansByPage = pages.map((p) => p.spans);
+    const byPage: PlacedBox[][] = pages.map(() => []);
+    for (const entry of boxes) {
+      const located = locateField(entry.field.value, entry.field.page, entry.field.bbox, spansByPage);
+      if (located !== null) byPage[located.page]!.push({ entry, bbox: located.bbox });
+    }
+    return byPage;
+  }, [boxes, pages]);
+
   useEffect(() => {
     let cancelled = false;
     let loadingTask: PDFDocumentLoadingTask | null = null;
@@ -90,7 +132,10 @@ function PdfView({ file, boxes, activeKey, onActivate }: { file: File } & Omit<P
         loadingTask = getDocument({ data: new Uint8Array(buffer) });
         const doc = await loadingTask.promise;
         const proxies = await Promise.all(
-          Array.from({ length: doc.numPages }, (_, i) => doc.getPage(i + 1)),
+          Array.from({ length: doc.numPages }, async (_, i) => {
+            const page = await doc.getPage(i + 1);
+            return { page, spans: await pageTextSpans(page) };
+          }),
         );
         if (cancelled) return;
         setPages(proxies);
@@ -108,10 +153,10 @@ function PdfView({ file, boxes, activeKey, onActivate }: { file: File } & Omit<P
 
   return (
     <div className="page-stack">
-      {pages.map((page, index) => (
+      {pages.map(({ page }, index) => (
         <div className="page" key={index}>
           <PdfCanvas page={page} />
-          <BoxLayer boxes={boxes} pageIndex={index} activeKey={activeKey} onActivate={onActivate} />
+          <BoxLayer placed={placedByPage[index]!} activeKey={activeKey} onActivate={onActivate} />
         </div>
       ))}
     </div>
@@ -143,34 +188,47 @@ function PdfCanvas({ page }: { page: PDFPageProxy }) {
   return <canvas ref={canvasRef} className="page-canvas" />;
 }
 
-function BoxLayer({ boxes, pageIndex, activeKey, onActivate }: PageBoxProps) {
-  const onPage = boxes.filter((box) => (box.field.page ?? 0) === pageIndex);
+function BoxLayer({
+  placed,
+  activeKey,
+  onActivate,
+}: {
+  placed: PlacedBox[];
+  activeKey: string | null;
+  onActivate: (key: string | null) => void;
+}) {
   return (
     <div className="box-layer">
-      {onPage.map((box) => (
-        <BBox key={box.key} entry={box} active={activeKey === box.key} onActivate={onActivate} />
+      {placed.map(({ entry, bbox }) => (
+        <BBox
+          key={entry.key}
+          entry={entry}
+          bbox={bbox}
+          active={activeKey === entry.key}
+          onActivate={onActivate}
+        />
       ))}
     </div>
   );
 }
 
 function BBox({
   entry,
+  bbox,
   active,
   onActivate,
 }: {
   entry: FieldEntry;
+  bbox: BBoxRect;
   active: boolean;
   onActivate: (key: string | null) => void;
 }) {
   const ref = useRef<HTMLDivElement>(null);
-  const { bbox } = entry.field;
 
   useEffect(() => {
     if (active) ref.current?.scrollIntoView({ block: 'nearest', behavior: 'smooth' });
   }, [active]);
 
-  if (bbox === null) return null;
   return (
     <div
       ref={ref}

apps/playground/src/client/components/ResultPanel.tsx

@@ -16,6 +16,9 @@ interface ResultPanelProps {
 
 export function ResultPanel(props: ResultPanelProps) {
   const { phase, live, result, error, hasFile, activeKey, onActivate } = props;
+  // A failed run may still carry a partial extraction — show it under the error.
+  const fields = result !== null ? result.fields : (error?.partial?.fields ?? null);
+  const usage = result !== null ? result.usage : (error?.partial?.usage ?? null);
 
   return (
     <div className="result-panel">
@@ -26,18 +29,18 @@ export function ResultPanel(props: ResultPanelProps) {
 
       <div className="result-body">
         {error !== null && <ErrorBanner error={error} />}
-        {error === null && result !== null && (
+        {fields !== null && (
           <div className="tree">
-            <FieldNode node={result.fields} path={[]} activeKey={activeKey} onActivate={onActivate} />
+            <FieldNode node={fields} path={[]} activeKey={activeKey} onActivate={onActivate} />
           </div>
         )}
-        {error === null && result === null && phase === 'running' && (
+        {error === null && fields === null && phase === 'running' && (
           <LiveList entries={live} activeKey={activeKey} onActivate={onActivate} />
         )}
-        {error === null && result === null && phase !== 'running' && <EmptyHint hasFile={hasFile} />}
+        {error === null && fields === null && phase !== 'running' && <EmptyHint hasFile={hasFile} />}
       </div>
 
-      {error === null && result !== null && <ResultFooter result={result} />}
+      {usage !== null && <ResultFooter usage={usage} issues={result?.issues ?? []} />}
     </div>
   );
 }
@@ -201,8 +204,7 @@ function EmptyHint({ hasFile }: { hasFile: boolean }) {
   );
 }
 
-function ResultFooter({ result }: { result: SerializedResult }) {
-  const { usage } = result;
+function ResultFooter({ usage, issues }: { usage: SerializedResult['usage']; issues: string[] }) {
   return (
     <div className="result-footer">
       <div className="stats">
@@ -212,13 +214,13 @@ function ResultFooter({ result }: { result: SerializedResult }) {
         <Stat label="Cost" value={usage.costUSD !== null ? formatUSD(usage.costUSD) : '—'} />
         <Stat label="Est. / 1k docs" value={usage.costUSD !== null ? formatUSD(usage.costUSD * 1000) : '—'} />
       </div>
-      {result.issues.length > 0 && (
+      {issues.length > 0 && (
         <details className="issues">
           <summary>
-            {result.issues.length} provenance {result.issues.length === 1 ? 'note' : 'notes'}
+            {issues.length} provenance {issues.length === 1 ? 'note' : 'notes'}
           </summary>
           <ul>
-            {result.issues.map((issue, i) => (
+            {issues.map((issue, i) => (
               <li key={i}>{issue}</li>
             ))}
           </ul>
@@ -250,19 +252,24 @@ const ERROR_TITLES: Record<string, string> = {
 
 function ErrorBanner({ error }: { error: ApiError }) {
   const title = (error.code !== null && ERROR_TITLES[error.code]) || error.name;
+  const paths = error.missingPaths ?? [];
   return (
     <div className="error-banner">
       <div className="error-title">{title}</div>
-      <div className="error-message">{error.message}</div>
-      {error.missingPaths !== undefined && error.missingPaths.length > 0 && (
+      {/* For missing-fields errors the message just repeats the paths list. */}
+      {paths.length === 0 && <div className="error-message">{error.message}</div>}
+      {paths.length > 0 && (
         <ul className="error-paths">
-          {error.missingPaths.map((path) => (
+          {paths.map((path) => (
             <li key={path}>
               <code>{path}</code>
             </li>
           ))}
         </ul>
       )}
+      {error.partial !== undefined && (
+        <div className="error-message">Everything that was extracted is shown below.</div>
+      )}
     </div>
   );
 }