diff --git a/dashboard/package-lock.json b/dashboard/package-lock.json index 345c73d2..341dce91 100644 --- a/dashboard/package-lock.json +++ b/dashboard/package-lock.json @@ -11,7 +11,8 @@ "highlight.js": "^11.11.1", "katex": "^0.16.27", "marked": "^17.0.1", - "mode-watcher": "^1.1.0" + "mode-watcher": "^1.1.0", + "pdfjs-dist": "^5.6.205" }, "devDependencies": { "@sveltejs/adapter-static": "^3.0.10", @@ -518,6 +519,256 @@ "@jridgewell/sourcemap-codec": "^1.4.14" } }, + "node_modules/@napi-rs/canvas": { + "version": "0.1.97", + "resolved": "https://registry.npmjs.org/@napi-rs/canvas/-/canvas-0.1.97.tgz", + "integrity": "sha512-8cFniXvrIEnVwuNSRCW9wirRZbHvrD3JVujdS2P5n5xiJZNZMOZcfOvJ1pb66c7jXMKHHglJEDVJGbm8XWFcXQ==", + "license": "MIT", + "optional": true, + "workspaces": [ + "e2e/*" + ], + "engines": { + "node": ">= 10" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/Brooooooklyn" + }, + "optionalDependencies": { + "@napi-rs/canvas-android-arm64": "0.1.97", + "@napi-rs/canvas-darwin-arm64": "0.1.97", + "@napi-rs/canvas-darwin-x64": "0.1.97", + "@napi-rs/canvas-linux-arm-gnueabihf": "0.1.97", + "@napi-rs/canvas-linux-arm64-gnu": "0.1.97", + "@napi-rs/canvas-linux-arm64-musl": "0.1.97", + "@napi-rs/canvas-linux-riscv64-gnu": "0.1.97", + "@napi-rs/canvas-linux-x64-gnu": "0.1.97", + "@napi-rs/canvas-linux-x64-musl": "0.1.97", + "@napi-rs/canvas-win32-arm64-msvc": "0.1.97", + "@napi-rs/canvas-win32-x64-msvc": "0.1.97" + } + }, + "node_modules/@napi-rs/canvas-android-arm64": { + "version": "0.1.97", + "resolved": "https://registry.npmjs.org/@napi-rs/canvas-android-arm64/-/canvas-android-arm64-0.1.97.tgz", + "integrity": "sha512-V1c/WVw+NzH8vk7ZK/O8/nyBSCQimU8sfMsB/9qeSvdkGKNU7+mxy/bIF0gTgeBFmHpj30S4E9WHMSrxXGQuVQ==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">= 10" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/Brooooooklyn" + } + }, + "node_modules/@napi-rs/canvas-darwin-arm64": { + "version": "0.1.97", + "resolved": "https://registry.npmjs.org/@napi-rs/canvas-darwin-arm64/-/canvas-darwin-arm64-0.1.97.tgz", + "integrity": "sha512-ok+SCEF4YejcxuJ9Rm+WWunHHpf2HmiPxfz6z1a/NFQECGXtsY7A4B8XocK1LmT1D7P174MzwPF9Wy3AUAwEPw==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">= 10" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/Brooooooklyn" + } + }, + "node_modules/@napi-rs/canvas-darwin-x64": { + "version": "0.1.97", + "resolved": "https://registry.npmjs.org/@napi-rs/canvas-darwin-x64/-/canvas-darwin-x64-0.1.97.tgz", + "integrity": "sha512-PUP6e6/UGlclUvAQNnuXCcnkpdUou6VYZfQOQxExLp86epOylmiwLkqXIvpFmjoTEDmPmXrI+coL/9EFU1gKPA==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">= 10" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/Brooooooklyn" + } + }, + "node_modules/@napi-rs/canvas-linux-arm-gnueabihf": { + "version": "0.1.97", + "resolved": "https://registry.npmjs.org/@napi-rs/canvas-linux-arm-gnueabihf/-/canvas-linux-arm-gnueabihf-0.1.97.tgz", + "integrity": "sha512-XyXH2L/cic8eTNtbrXCcvqHtMX/nEOxN18+7rMrAM2XtLYC/EB5s0wnO1FsLMWmK+04ZSLN9FBGipo7kpIkcOw==", + "cpu": [ + "arm" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/Brooooooklyn" + } + }, + "node_modules/@napi-rs/canvas-linux-arm64-gnu": { + "version": "0.1.97", + "resolved": "https://registry.npmjs.org/@napi-rs/canvas-linux-arm64-gnu/-/canvas-linux-arm64-gnu-0.1.97.tgz", + "integrity": "sha512-Kuq/M3djq0K8ktgz6nPlK7Ne5d4uWeDxPpyKWOjWDK2RIOhHVtLtyLiJw2fuldw7Vn4mhw05EZXCEr4Q76rs9w==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/Brooooooklyn" + } + }, + "node_modules/@napi-rs/canvas-linux-arm64-musl": { + "version": "0.1.97", + "resolved": "https://registry.npmjs.org/@napi-rs/canvas-linux-arm64-musl/-/canvas-linux-arm64-musl-0.1.97.tgz", + "integrity": "sha512-kKmSkQVnWeqg7qdsiXvYxKhAFuHz3tkBjW/zyQv5YKUPhotpaVhpBGv5LqCngzyuRV85SXoe+OFj+Tv0a0QXkQ==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/Brooooooklyn" + } + }, + "node_modules/@napi-rs/canvas-linux-riscv64-gnu": { + "version": "0.1.97", + "resolved": "https://registry.npmjs.org/@napi-rs/canvas-linux-riscv64-gnu/-/canvas-linux-riscv64-gnu-0.1.97.tgz", + "integrity": "sha512-Jc7I3A51jnEOIAXeLsN/M/+Z28LUeakcsXs07FLq9prXc0eYOtVwsDEv913Gr+06IRo34gJJVgT0TXvmz+N2VA==", + "cpu": [ + "riscv64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/Brooooooklyn" + } + }, + "node_modules/@napi-rs/canvas-linux-x64-gnu": { + "version": "0.1.97", + "resolved": "https://registry.npmjs.org/@napi-rs/canvas-linux-x64-gnu/-/canvas-linux-x64-gnu-0.1.97.tgz", + "integrity": "sha512-iDUBe7AilfuBSRbSa8/IGX38Mf+iCSBqoVKLSQ5XaY2JLOaqz1TVyPFEyIck7wT6mRQhQt5sN6ogfjIDfi74tg==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/Brooooooklyn" + } + }, + "node_modules/@napi-rs/canvas-linux-x64-musl": { + "version": "0.1.97", + "resolved": "https://registry.npmjs.org/@napi-rs/canvas-linux-x64-musl/-/canvas-linux-x64-musl-0.1.97.tgz", + "integrity": "sha512-AKLFd/v0Z5fvgqBDqhvqtAdx+fHMJ5t9JcUNKq4FIZ5WH+iegGm8HPdj00NFlCSnm83Fp3Ln8I2f7uq1aIiWaA==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/Brooooooklyn" + } + }, + "node_modules/@napi-rs/canvas-win32-arm64-msvc": { + "version": "0.1.97", + "resolved": "https://registry.npmjs.org/@napi-rs/canvas-win32-arm64-msvc/-/canvas-win32-arm64-msvc-0.1.97.tgz", + "integrity": "sha512-u883Yr6A6fO7Vpsy9YE4FVCIxzzo5sO+7pIUjjoDLjS3vQaNMkVzx5bdIpEL+ob+gU88WDK4VcxYMZ6nmnoX9A==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">= 10" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/Brooooooklyn" + } + }, + "node_modules/@napi-rs/canvas-win32-x64-msvc": { + "version": "0.1.97", + "resolved": "https://registry.npmjs.org/@napi-rs/canvas-win32-x64-msvc/-/canvas-win32-x64-msvc-0.1.97.tgz", + "integrity": "sha512-sWtD2EE3fV0IzN+iiQUqr/Q1SwqWhs2O1FKItFlxtdDkikpEj5g7DKQpY3x55H/MAOnL8iomnlk3mcEeGiUMoQ==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">= 10" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/Brooooooklyn" + } + }, "node_modules/@polka/url": { "version": "1.0.0-next.29", "resolved": "https://registry.npmjs.org/@polka/url/-/url-1.0.0-next.29.tgz", @@ -2635,6 +2886,26 @@ "node": "^10 || ^12 || ^13.7 || ^14 || >=15.0.1" } }, + "node_modules/node-readable-to-web-readable-stream": { + "version": "0.4.2", + "resolved": "https://registry.npmjs.org/node-readable-to-web-readable-stream/-/node-readable-to-web-readable-stream-0.4.2.tgz", + "integrity": "sha512-/cMZNI34v//jUTrI+UIo4ieHAB5EZRY/+7OmXZgBxaWBMcW2tGdceIw06RFxWxrKZ5Jp3sI2i5TsRo+CBhtVLQ==", + "license": "MIT", + "optional": true + }, + "node_modules/pdfjs-dist": { + "version": "5.6.205", + "resolved": "https://registry.npmjs.org/pdfjs-dist/-/pdfjs-dist-5.6.205.tgz", + "integrity": "sha512-tlUj+2IDa7G1SbvBNN74UHRLJybZDWYom+k6p5KIZl7huBvsA4APi6mKL+zCxd3tLjN5hOOEE9Tv7VdzO88pfg==", + "license": "Apache-2.0", + "engines": { + "node": ">=20.19.0 || >=22.13.0 || >=24" + }, + "optionalDependencies": { + "@napi-rs/canvas": "^0.1.96", + "node-readable-to-web-readable-stream": "^0.4.2" + } + }, "node_modules/picocolors": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.1.1.tgz", diff --git a/dashboard/package.json b/dashboard/package.json index 45ec739c..08179b5c 100644 --- a/dashboard/package.json +++ b/dashboard/package.json @@ -31,6 +31,7 @@ "highlight.js": "^11.11.1", "katex": "^0.16.27", "marked": "^17.0.1", - "mode-watcher": "^1.1.0" + "mode-watcher": "^1.1.0", + "pdfjs-dist": "^5.6.205" } } diff --git a/dashboard/src/lib/components/ChatMessages.svelte b/dashboard/src/lib/components/ChatMessages.svelte index c716ec3b..46c20694 100644 --- a/dashboard/src/lib/components/ChatMessages.svelte +++ b/dashboard/src/lib/components/ChatMessages.svelte @@ -139,6 +139,8 @@ return "🖼"; case "text": return "📄"; + case "pdf": + return "📑"; default: return "📎"; } diff --git a/dashboard/src/lib/stores/app.svelte.ts b/dashboard/src/lib/stores/app.svelte.ts index b43776cc..352a4c07 100644 --- a/dashboard/src/lib/stores/app.svelte.ts +++ b/dashboard/src/lib/stores/app.svelte.ts @@ -257,11 +257,12 @@ interface RawStateResponse { } export interface MessageAttachment { - type: "image" | "text" | "file" | "generated-image"; + type: "image" | "text" | "file" | "generated-image" | "pdf"; name: string; content?: string; preview?: string; mimeType?: string; + pageImages?: string[]; } export interface TopLogprob { @@ -2242,6 +2243,7 @@ class AppStore { type: string; textContent?: string; preview?: string; + pageImages?: string[]; }[], enableThinking?: boolean | null, ): Promise { @@ -2277,6 +2279,20 @@ class AppStore { preview: file.preview, mimeType: file.type, }); + } else if ( + file.pageImages || + (file.textContent && file.type === "application/pdf") + ) { + attachments.push({ + type: "pdf", + name: file.name, + content: file.textContent, + pageImages: file.pageImages, + mimeType: file.type, + }); + if (file.textContent) { + fileContext += `\n\n[File: ${file.name}]\n\`\`\`\n${file.textContent}\n\`\`\``; + } } else if (file.textContent) { attachments.push({ type: "text", @@ -2344,12 +2360,14 @@ class AppStore { const apiMessages = [ systemPrompt, ...targetConversation.messages.slice(0, -1).map((m) => { - // Check if this message has image attachments - const imageAttachments = m.attachments?.filter( - (a) => a.type === "image" && a.preview, + // Check if this message has image or PDF attachments + const visualAttachments = m.attachments?.filter( + (a) => + (a.type === "image" && a.preview) || + (a.type === "pdf" && a.pageImages?.length), ); - if (imageAttachments && imageAttachments.length > 0) { + if (visualAttachments && visualAttachments.length > 0) { // Build multimodal content array (OpenAI vision format) const contentParts: Array< | { type: "text"; text: string } @@ -2357,20 +2375,30 @@ class AppStore { > = []; // Add image parts first - for (const img of imageAttachments) { - if (img.preview) { + for (const att of visualAttachments) { + if (att.type === "image" && att.preview) { contentParts.push({ type: "image_url", - image_url: { url: img.preview }, + image_url: { url: att.preview }, }); + } else if (att.type === "pdf" && att.pageImages) { + for (const pageImg of att.pageImages) { + contentParts.push({ + type: "image_url", + image_url: { url: pageImg }, + }); + } } } - // Build text content including any text file attachments + // Build text content including any text/pdf file attachments let textContent = m.content; if (m.attachments) { for (const attachment of m.attachments) { - if (attachment.type === "text" && attachment.content) { + if ( + (attachment.type === "text" || attachment.type === "pdf") && + attachment.content + ) { textContent += `\n\n[File: ${attachment.name}]\n\`\`\`\n${attachment.content}\n\`\`\``; } } @@ -2389,10 +2417,13 @@ class AppStore { // Text-only message (original path) let msgContent = m.content; - // Add text attachments as context + // Add text/pdf attachments as context if (m.attachments) { for (const attachment of m.attachments) { - if (attachment.type === "text" && attachment.content) { + if ( + (attachment.type === "text" || attachment.type === "pdf") && + attachment.content + ) { msgContent += `\n\n[File: ${attachment.name}]\n\`\`\`\n${attachment.content}\n\`\`\``; } } @@ -3347,6 +3378,7 @@ export const sendMessage = ( type: string; textContent?: string; preview?: string; + pageImages?: string[]; }[], enableThinking?: boolean | null, ) => appStore.sendMessage(content, files, enableThinking); diff --git a/dashboard/src/lib/types/files.ts b/dashboard/src/lib/types/files.ts index 4b11f128..d2231d57 100644 --- a/dashboard/src/lib/types/files.ts +++ b/dashboard/src/lib/types/files.ts @@ -2,6 +2,15 @@ * File attachment types for the chat interface */ +import { getDocument, GlobalWorkerOptions, version } from "pdfjs-dist"; +import type { DocumentInitParameters } from "pdfjs-dist/types/src/display/api"; + +GlobalWorkerOptions.workerSrc = `https://cdn.jsdelivr.net/npm/pdfjs-dist@${version}/build/pdf.worker.mjs`; + +const PDF_PAGE_SCALE = 2.0; +const PDF_MAX_PAGES = 20; +const PDF_MAX_TEXT_CHARS = 100_000; + export interface ChatUploadedFile { id: string; name: string; @@ -10,6 +19,7 @@ export interface ChatUploadedFile { file: File; preview?: string; textContent?: string; + pageImages?: string[]; } export interface ChatAttachment { @@ -194,6 +204,58 @@ export function readFileAsText(file: File): Promise { }); } +async function extractPdfContent( + file: File, +): Promise<{ text: string; pageImages: string[] }> { + const arrayBuffer = await file.arrayBuffer(); + const pdf = await getDocument({ + data: new Uint8Array(arrayBuffer), + useSystemFonts: true, + } as DocumentInitParameters).promise; + + const numPages = Math.min(pdf.numPages, PDF_MAX_PAGES); + const pageTexts: string[] = []; + const pageImages: string[] = []; + + for (let i = 1; i <= numPages; i++) { + const page = await pdf.getPage(i); + + const content = await page.getTextContent(); + const strings = content.items + .filter((item: any) => "str" in item) + .map((item: any) => item.str as string); + pageTexts.push(strings.join(" ")); + + const viewport = page.getViewport({ scale: PDF_PAGE_SCALE }); + const canvas = new OffscreenCanvas(viewport.width, viewport.height); + const ctx = canvas.getContext("2d"); + if (ctx) { + await page.render({ canvasContext: ctx as any, viewport }).promise; + const blob = await canvas.convertToBlob({ + type: "image/jpeg", + quality: 0.8, + }); + const reader = new FileReader(); + const dataUrl = await new Promise((resolve, reject) => { + reader.onload = () => resolve(reader.result as string); + reader.onerror = () => reject(reader.error); + reader.readAsDataURL(blob); + }); + pageImages.push(dataUrl); + } + } + + let text = pageTexts.join("\n\n").trim(); + if (text.length > PDF_MAX_TEXT_CHARS) { + text = text.slice(0, PDF_MAX_TEXT_CHARS) + "\n\n[truncated]"; + } + if (pdf.numPages > PDF_MAX_PAGES) { + text += `\n\n[showing ${PDF_MAX_PAGES} of ${pdf.numPages} pages]`; + } + + return { text, pageImages }; +} + /** * Process uploaded files into ChatUploadedFile format */ @@ -223,7 +285,12 @@ export async function processUploadedFiles( const textContent = await readFileAsText(file); results.push({ ...base, textContent }); } else if (category === "pdf") { - results.push(base); + const { text, pageImages } = await extractPdfContent(file); + results.push({ + ...base, + textContent: text || undefined, + pageImages: pageImages.length > 0 ? pageImages : undefined, + }); } else if (category === "audio") { const preview = await readFileAsDataURL(file); results.push({ ...base, preview });