electron-rare/exo
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Support PDFs in dashboard (#1822)

Browse Source 
Like ChatGPT does, we now send both the extracted text and the image of
each PDF page.
This commit is contained in:
rltakashige
2026-03-31 18:25:40 +01:00
committed by GitHub
parent d9ed943034
commit 4688adb5d2
5 changed files with 388 additions and 15 deletions
Download Patch File Download Diff File Expand all files Collapse all files
dashboard/package-lock.json
Generated
+272 -1
View File
@@ -11,7 +11,8 @@
"highlight.js": "^11.11.1",
"katex": "^0.16.27",
"marked": "^17.0.1",
"mode-watcher": "^1.1.0"
"mode-watcher": "^1.1.0",
"pdfjs-dist": "^5.6.205"
},
"devDependencies": {
"@sveltejs/adapter-static": "^3.0.10",
@@ -518,6 +519,256 @@
"@jridgewell/sourcemap-codec": "^1.4.14"
}
},
"node_modules/@napi-rs/canvas": {
"version": "0.1.97",
"resolved": "https://registry.npmjs.org/@napi-rs/canvas/-/canvas-0.1.97.tgz",
"integrity": "sha512-8cFniXvrIEnVwuNSRCW9wirRZbHvrD3JVujdS2P5n5xiJZNZMOZcfOvJ1pb66c7jXMKHHglJEDVJGbm8XWFcXQ==",
"license": "MIT",
"optional": true,
"workspaces": [
"e2e/*"
],
"engines": {
"node": ">= 10"
},
"funding": {
"type": "github",
"url": "https://github.com/sponsors/Brooooooklyn"
},
"optionalDependencies": {
"@napi-rs/canvas-android-arm64": "0.1.97",
"@napi-rs/canvas-darwin-arm64": "0.1.97",
"@napi-rs/canvas-darwin-x64": "0.1.97",
"@napi-rs/canvas-linux-arm-gnueabihf": "0.1.97",
"@napi-rs/canvas-linux-arm64-gnu": "0.1.97",
"@napi-rs/canvas-linux-arm64-musl": "0.1.97",
"@napi-rs/canvas-linux-riscv64-gnu": "0.1.97",
"@napi-rs/canvas-linux-x64-gnu": "0.1.97",
"@napi-rs/canvas-linux-x64-musl": "0.1.97",
"@napi-rs/canvas-win32-arm64-msvc": "0.1.97",
"@napi-rs/canvas-win32-x64-msvc": "0.1.97"
}
},
"node_modules/@napi-rs/canvas-android-arm64": {
"version": "0.1.97",
"resolved": "https://registry.npmjs.org/@napi-rs/canvas-android-arm64/-/canvas-android-arm64-0.1.97.tgz",
"integrity": "sha512-V1c/WVw+NzH8vk7ZK/O8/nyBSCQimU8sfMsB/9qeSvdkGKNU7+mxy/bIF0gTgeBFmHpj30S4E9WHMSrxXGQuVQ==",
"cpu": [
"arm64"
],
"license": "MIT",
"optional": true,
"os": [
"android"
],
"engines": {
"node": ">= 10"
},
"funding": {
"type": "github",
"url": "https://github.com/sponsors/Brooooooklyn"
}
},
"node_modules/@napi-rs/canvas-darwin-arm64": {
"version": "0.1.97",
"resolved": "https://registry.npmjs.org/@napi-rs/canvas-darwin-arm64/-/canvas-darwin-arm64-0.1.97.tgz",
"integrity": "sha512-ok+SCEF4YejcxuJ9Rm+WWunHHpf2HmiPxfz6z1a/NFQECGXtsY7A4B8XocK1LmT1D7P174MzwPF9Wy3AUAwEPw==",
"cpu": [
"arm64"
],
"license": "MIT",
"optional": true,
"os": [
"darwin"
],
"engines": {
"node": ">= 10"
},
"funding": {
"type": "github",
"url": "https://github.com/sponsors/Brooooooklyn"
}
},
"node_modules/@napi-rs/canvas-darwin-x64": {
"version": "0.1.97",
"resolved": "https://registry.npmjs.org/@napi-rs/canvas-darwin-x64/-/canvas-darwin-x64-0.1.97.tgz",
"integrity": "sha512-PUP6e6/UGlclUvAQNnuXCcnkpdUou6VYZfQOQxExLp86epOylmiwLkqXIvpFmjoTEDmPmXrI+coL/9EFU1gKPA==",
"cpu": [
"x64"
],
"license": "MIT",
"optional": true,
"os": [
"darwin"
],
"engines": {
"node": ">= 10"
},
"funding": {
"type": "github",
"url": "https://github.com/sponsors/Brooooooklyn"
}
},
"node_modules/@napi-rs/canvas-linux-arm-gnueabihf": {
"version": "0.1.97",
"resolved": "https://registry.npmjs.org/@napi-rs/canvas-linux-arm-gnueabihf/-/canvas-linux-arm-gnueabihf-0.1.97.tgz",
"integrity": "sha512-XyXH2L/cic8eTNtbrXCcvqHtMX/nEOxN18+7rMrAM2XtLYC/EB5s0wnO1FsLMWmK+04ZSLN9FBGipo7kpIkcOw==",
"cpu": [
"arm"
],
"license": "MIT",
"optional": true,
"os": [
"linux"
],
"engines": {
"node": ">= 10"
},
"funding": {
"type": "github",
"url": "https://github.com/sponsors/Brooooooklyn"
}
},
"node_modules/@napi-rs/canvas-linux-arm64-gnu": {
"version": "0.1.97",
"resolved": "https://registry.npmjs.org/@napi-rs/canvas-linux-arm64-gnu/-/canvas-linux-arm64-gnu-0.1.97.tgz",
"integrity": "sha512-Kuq/M3djq0K8ktgz6nPlK7Ne5d4uWeDxPpyKWOjWDK2RIOhHVtLtyLiJw2fuldw7Vn4mhw05EZXCEr4Q76rs9w==",
"cpu": [
"arm64"
],
"license": "MIT",
"optional": true,
"os": [
"linux"
],
"engines": {
"node": ">= 10"
},
"funding": {
"type": "github",
"url": "https://github.com/sponsors/Brooooooklyn"
}
},
"node_modules/@napi-rs/canvas-linux-arm64-musl": {
"version": "0.1.97",
"resolved": "https://registry.npmjs.org/@napi-rs/canvas-linux-arm64-musl/-/canvas-linux-arm64-musl-0.1.97.tgz",
"integrity": "sha512-kKmSkQVnWeqg7qdsiXvYxKhAFuHz3tkBjW/zyQv5YKUPhotpaVhpBGv5LqCngzyuRV85SXoe+OFj+Tv0a0QXkQ==",
"cpu": [
"arm64"
],
"license": "MIT",
"optional": true,
"os": [
"linux"
],
"engines": {
"node": ">= 10"
},
"funding": {
"type": "github",
"url": "https://github.com/sponsors/Brooooooklyn"
}
},
"node_modules/@napi-rs/canvas-linux-riscv64-gnu": {
"version": "0.1.97",
"resolved": "https://registry.npmjs.org/@napi-rs/canvas-linux-riscv64-gnu/-/canvas-linux-riscv64-gnu-0.1.97.tgz",
"integrity": "sha512-Jc7I3A51jnEOIAXeLsN/M/+Z28LUeakcsXs07FLq9prXc0eYOtVwsDEv913Gr+06IRo34gJJVgT0TXvmz+N2VA==",
"cpu": [
"riscv64"
],
"license": "MIT",
"optional": true,
"os": [
"linux"
],
"engines": {
"node": ">= 10"
},
"funding": {
"type": "github",
"url": "https://github.com/sponsors/Brooooooklyn"
}
},
"node_modules/@napi-rs/canvas-linux-x64-gnu": {
"version": "0.1.97",
"resolved": "https://registry.npmjs.org/@napi-rs/canvas-linux-x64-gnu/-/canvas-linux-x64-gnu-0.1.97.tgz",
"integrity": "sha512-iDUBe7AilfuBSRbSa8/IGX38Mf+iCSBqoVKLSQ5XaY2JLOaqz1TVyPFEyIck7wT6mRQhQt5sN6ogfjIDfi74tg==",
"cpu": [
"x64"
],
"license": "MIT",
"optional": true,
"os": [
"linux"
],
"engines": {
"node": ">= 10"
},
"funding": {
"type": "github",
"url": "https://github.com/sponsors/Brooooooklyn"
}
},
"node_modules/@napi-rs/canvas-linux-x64-musl": {
"version": "0.1.97",
"resolved": "https://registry.npmjs.org/@napi-rs/canvas-linux-x64-musl/-/canvas-linux-x64-musl-0.1.97.tgz",
"integrity": "sha512-AKLFd/v0Z5fvgqBDqhvqtAdx+fHMJ5t9JcUNKq4FIZ5WH+iegGm8HPdj00NFlCSnm83Fp3Ln8I2f7uq1aIiWaA==",
"cpu": [
"x64"
],
"license": "MIT",
"optional": true,
"os": [
"linux"
],
"engines": {
"node": ">= 10"
},
"funding": {
"type": "github",
"url": "https://github.com/sponsors/Brooooooklyn"
}
},
"node_modules/@napi-rs/canvas-win32-arm64-msvc": {
"version": "0.1.97",
"resolved": "https://registry.npmjs.org/@napi-rs/canvas-win32-arm64-msvc/-/canvas-win32-arm64-msvc-0.1.97.tgz",
"integrity": "sha512-u883Yr6A6fO7Vpsy9YE4FVCIxzzo5sO+7pIUjjoDLjS3vQaNMkVzx5bdIpEL+ob+gU88WDK4VcxYMZ6nmnoX9A==",
"cpu": [
"arm64"
],
"license": "MIT",
"optional": true,
"os": [
"win32"
],
"engines": {
"node": ">= 10"
},
"funding": {
"type": "github",
"url": "https://github.com/sponsors/Brooooooklyn"
}
},
"node_modules/@napi-rs/canvas-win32-x64-msvc": {
"version": "0.1.97",
"resolved": "https://registry.npmjs.org/@napi-rs/canvas-win32-x64-msvc/-/canvas-win32-x64-msvc-0.1.97.tgz",
"integrity": "sha512-sWtD2EE3fV0IzN+iiQUqr/Q1SwqWhs2O1FKItFlxtdDkikpEj5g7DKQpY3x55H/MAOnL8iomnlk3mcEeGiUMoQ==",
"cpu": [
"x64"
],
"license": "MIT",
"optional": true,
"os": [
"win32"
],
"engines": {
"node": ">= 10"
},
"funding": {
"type": "github",
"url": "https://github.com/sponsors/Brooooooklyn"
}
},
"node_modules/@polka/url": {
"version": "1.0.0-next.29",
"resolved": "https://registry.npmjs.org/@polka/url/-/url-1.0.0-next.29.tgz",
@@ -2635,6 +2886,26 @@
"node": "^10 || ^12 || ^13.7 || ^14 || >=15.0.1"
}
},
"node_modules/node-readable-to-web-readable-stream": {
"version": "0.4.2",
"resolved": "https://registry.npmjs.org/node-readable-to-web-readable-stream/-/node-readable-to-web-readable-stream-0.4.2.tgz",
"integrity": "sha512-/cMZNI34v//jUTrI+UIo4ieHAB5EZRY/+7OmXZgBxaWBMcW2tGdceIw06RFxWxrKZ5Jp3sI2i5TsRo+CBhtVLQ==",
"license": "MIT",
"optional": true
},
"node_modules/pdfjs-dist": {
"version": "5.6.205",
"resolved": "https://registry.npmjs.org/pdfjs-dist/-/pdfjs-dist-5.6.205.tgz",
"integrity": "sha512-tlUj+2IDa7G1SbvBNN74UHRLJybZDWYom+k6p5KIZl7huBvsA4APi6mKL+zCxd3tLjN5hOOEE9Tv7VdzO88pfg==",
"license": "Apache-2.0",
"engines": {
"node": ">=20.19.0 || >=22.13.0 || >=24"
},
"optionalDependencies": {
"@napi-rs/canvas": "^0.1.96",
"node-readable-to-web-readable-stream": "^0.4.2"
}
},
"node_modules/picocolors": {
"version": "1.1.1",
"resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.1.1.tgz",
dashboard/package.json
+2 -1
View File
@@ -31,6 +31,7 @@
"highlight.js": "^11.11.1",
"katex": "^0.16.27",
"marked": "^17.0.1",
"mode-watcher": "^1.1.0"
"mode-watcher": "^1.1.0",
"pdfjs-dist": "^5.6.205"
}
}
dashboard/src/lib/components/ChatMessages.svelte
+2
View File
@@ -139,6 +139,8 @@
return "🖼";
case "text":
return "📄";
case "pdf":
return "📑";
default:
return "📎";
}
dashboard/src/lib/stores/app.svelte.ts
+44 -12
View File
@@ -257,11 +257,12 @@ interface RawStateResponse {
}
export interface MessageAttachment {
type: "image" | "text" | "file" | "generated-image";
type: "image" | "text" | "file" | "generated-image" | "pdf";
name: string;
content?: string;
preview?: string;
mimeType?: string;
pageImages?: string[];
}
export interface TopLogprob {
@@ -2242,6 +2243,7 @@ class AppStore {
type: string;
textContent?: string;
preview?: string;
pageImages?: string[];
}[],
enableThinking?: boolean | null,
): Promise<void> {
@@ -2277,6 +2279,20 @@ class AppStore {
preview: file.preview,
mimeType: file.type,
});
} else if (
file.pageImages ||
(file.textContent && file.type === "application/pdf")
) {
attachments.push({
type: "pdf",
name: file.name,
content: file.textContent,
pageImages: file.pageImages,
mimeType: file.type,
});
if (file.textContent) {
fileContext += `\n\n[File: ${file.name}]\n\`\`\`\n${file.textContent}\n\`\`\``;
}
} else if (file.textContent) {
attachments.push({
type: "text",
@@ -2344,12 +2360,14 @@ class AppStore {
const apiMessages = [
systemPrompt,
...targetConversation.messages.slice(0, -1).map((m) => {
// Check if this message has image attachments
const imageAttachments = m.attachments?.filter(
(a) => a.type === "image" && a.preview,
// Check if this message has image or PDF attachments
const visualAttachments = m.attachments?.filter(
(a) =>
(a.type === "image" && a.preview) ||
(a.type === "pdf" && a.pageImages?.length),
);
if (imageAttachments && imageAttachments.length > 0) {
if (visualAttachments && visualAttachments.length > 0) {
// Build multimodal content array (OpenAI vision format)
const contentParts: Array<
| { type: "text"; text: string }
@@ -2357,20 +2375,30 @@ class AppStore {
> = [];
// Add image parts first
for (const img of imageAttachments) {
if (img.preview) {
for (const att of visualAttachments) {
if (att.type === "image" && att.preview) {
contentParts.push({
type: "image_url",
image_url: { url: img.preview },
image_url: { url: att.preview },
});
} else if (att.type === "pdf" && att.pageImages) {
for (const pageImg of att.pageImages) {
contentParts.push({
type: "image_url",
image_url: { url: pageImg },
});
}
}
}
// Build text content including any text file attachments
// Build text content including any text/pdf file attachments
let textContent = m.content;
if (m.attachments) {
for (const attachment of m.attachments) {
if (attachment.type === "text" && attachment.content) {
if (
(attachment.type === "text" || attachment.type === "pdf") &&
attachment.content
) {
textContent += `\n\n[File: ${attachment.name}]\n\`\`\`\n${attachment.content}\n\`\`\``;
}
}
@@ -2389,10 +2417,13 @@ class AppStore {
// Text-only message (original path)
let msgContent = m.content;
// Add text attachments as context
// Add text/pdf attachments as context
if (m.attachments) {
for (const attachment of m.attachments) {
if (attachment.type === "text" && attachment.content) {
if (
(attachment.type === "text" || attachment.type === "pdf") &&
attachment.content
) {
msgContent += `\n\n[File: ${attachment.name}]\n\`\`\`\n${attachment.content}\n\`\`\``;
}
}
@@ -3347,6 +3378,7 @@ export const sendMessage = (
type: string;
textContent?: string;
preview?: string;
pageImages?: string[];
}[],
enableThinking?: boolean | null,
) => appStore.sendMessage(content, files, enableThinking);
dashboard/src/lib/types/files.ts
+68 -1
View File
@@ -2,6 +2,15 @@
* File attachment types for the chat interface
*/
import { getDocument, GlobalWorkerOptions, version } from "pdfjs-dist";
import type { DocumentInitParameters } from "pdfjs-dist/types/src/display/api";
GlobalWorkerOptions.workerSrc = `https://cdn.jsdelivr.net/npm/pdfjs-dist@${version}/build/pdf.worker.mjs`;
const PDF_PAGE_SCALE = 2.0;
const PDF_MAX_PAGES = 20;
const PDF_MAX_TEXT_CHARS = 100_000;
export interface ChatUploadedFile {
id: string;
name: string;
@@ -10,6 +19,7 @@ export interface ChatUploadedFile {
file: File;
preview?: string;
textContent?: string;
pageImages?: string[];
}
export interface ChatAttachment {
@@ -194,6 +204,58 @@ export function readFileAsText(file: File): Promise<string> {
});
}
async function extractPdfContent(
file: File,
): Promise<{ text: string; pageImages: string[] }> {
const arrayBuffer = await file.arrayBuffer();
const pdf = await getDocument({
data: new Uint8Array(arrayBuffer),
useSystemFonts: true,
} as DocumentInitParameters).promise;
const numPages = Math.min(pdf.numPages, PDF_MAX_PAGES);
const pageTexts: string[] = [];
const pageImages: string[] = [];
for (let i = 1; i <= numPages; i++) {
const page = await pdf.getPage(i);
const content = await page.getTextContent();
const strings = content.items
.filter((item: any) => "str" in item)
.map((item: any) => item.str as string);
pageTexts.push(strings.join(" "));
const viewport = page.getViewport({ scale: PDF_PAGE_SCALE });
const canvas = new OffscreenCanvas(viewport.width, viewport.height);
const ctx = canvas.getContext("2d");
if (ctx) {
await page.render({ canvasContext: ctx as any, viewport }).promise;
const blob = await canvas.convertToBlob({
type: "image/jpeg",
quality: 0.8,
});
const reader = new FileReader();
const dataUrl = await new Promise<string>((resolve, reject) => {
reader.onload = () => resolve(reader.result as string);
reader.onerror = () => reject(reader.error);
reader.readAsDataURL(blob);
});
pageImages.push(dataUrl);
}
}
let text = pageTexts.join("\n\n").trim();
if (text.length > PDF_MAX_TEXT_CHARS) {
text = text.slice(0, PDF_MAX_TEXT_CHARS) + "\n\n[truncated]";
}
if (pdf.numPages > PDF_MAX_PAGES) {
text += `\n\n[showing ${PDF_MAX_PAGES} of ${pdf.numPages} pages]`;
}
return { text, pageImages };
}
/**
* Process uploaded files into ChatUploadedFile format
*/
@@ -223,7 +285,12 @@ export async function processUploadedFiles(
const textContent = await readFileAsText(file);
results.push({ ...base, textContent });
} else if (category === "pdf") {
results.push(base);
const { text, pageImages } = await extractPdfContent(file);
results.push({
...base,
textContent: text || undefined,
pageImages: pageImages.length > 0 ? pageImages : undefined,
});
} else if (category === "audio") {
const preview = await readFileAsDataURL(file);
results.push({ ...base, preview });