diff --git a/CHANGELOG.md b/CHANGELOG.md index c4d59f85..66349b98 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,7 @@ and this project adheres to - ✨ Import of documents #1609 - 🚨(CI) gives warning if theme not updated #1811 - 🔧(project) add DJANGO_EMAIL_URL_APP environment variable #1825 +- ✨(frontend) integrate new Blocknote AI feature #1016 ### Changed diff --git a/src/frontend/apps/e2e/__tests__/app-impress/config.spec.ts b/src/frontend/apps/e2e/__tests__/app-impress/config.spec.ts index 44470fbb..239159e1 100644 --- a/src/frontend/apps/e2e/__tests__/app-impress/config.spec.ts +++ b/src/frontend/apps/e2e/__tests__/app-impress/config.spec.ts @@ -93,9 +93,7 @@ test.describe('Config', () => { expect( await page.locator('button[data-test="convertMarkdown"]').count(), ).toBe(1); - expect(await page.locator('button[data-test="ai-actions"]').count()).toBe( - 0, - ); + await expect(page.getByRole('button', { name: 'Ask AI' })).toBeHidden(); }); test('it checks that Crisp is trying to init from config endpoint', async ({ diff --git a/src/frontend/apps/e2e/__tests__/app-impress/doc-editor.spec.ts b/src/frontend/apps/e2e/__tests__/app-impress/doc-editor.spec.ts index e0ea8f59..e4f78671 100644 --- a/src/frontend/apps/e2e/__tests__/app-impress/doc-editor.spec.ts +++ b/src/frontend/apps/e2e/__tests__/app-impress/doc-editor.spec.ts @@ -389,6 +389,104 @@ test.describe('Doc Editor', () => { await expect(image).toHaveAttribute('aria-hidden', 'true'); }); + test('it checks the AI feature', async ({ page, browserName }) => { + await page.route(/.*\/ai-proxy\//, async (route) => { + const request = route.request(); + if (request.method().includes('POST')) { + await route.fulfill({ + json: { + id: 'chatcmpl-b1e7a9e456ca41f78fec130d552a6bf5', + choices: [ + { + finish_reason: 'stop', + index: 0, + logprobs: null, + message: { + content: '', + refusal: null, + role: 'assistant', + annotations: null, + audio: null, + function_call: null, + tool_calls: [ + { + id: 'chatcmpl-tool-2e3567dfecf94a4c85e27a3528337718', + function: { + arguments: + '{"operations": [{"type": "update", "id": "initialBlockId$", "block": "

Bonjour le monde

"}]}', + name: 'json', + }, + type: 'function', + }, + ], + reasoning_content: null, + }, + stop_reason: null, + }, + ], + created: 1749549477, + model: 'neuralmagic/Meta-Llama-3.1-70B-Instruct-FP8', + object: 'chat.completion', + service_tier: null, + system_fingerprint: null, + usage: { + completion_tokens: 0, + prompt_tokens: 204, + total_tokens: 204, + completion_tokens_details: null, + prompt_tokens_details: null, + details: [ + { + id: 'chatcmpl-b1e7a9e456ca41f78fec130d552a6bf5', + model: 'neuralmagic/Meta-Llama-3.1-70B-Instruct-FP8', + prompt_tokens: 204, + completion_tokens: 0, + total_tokens: 204, + }, + ], + }, + prompt_logprobs: null, + }, + }); + } else { + await route.continue(); + } + }); + + await createDoc(page, 'doc-ai', browserName, 1); + + await page.locator('.bn-block-outer').last().fill('Hello World'); + + const editor = page.locator('.ProseMirror'); + await editor.getByText('Hello World').selectText(); + + // Check from toolbar + await page.getByRole('button', { name: 'Ask AI' }).click(); + + await expect( + page.getByRole('option', { name: 'Improve Writing' }), + ).toBeVisible(); + await expect( + page.getByRole('option', { name: 'Fix Spelling' }), + ).toBeVisible(); + await expect(page.getByRole('option', { name: 'Translate' })).toBeVisible(); + + await page.getByRole('option', { name: 'Translate' }).click(); + await page.getByPlaceholder('Ask AI anything…').fill('French'); + await page.getByPlaceholder('Ask AI anything…').press('Enter'); + await expect(editor.getByText('Docs AI')).toBeVisible(); + await page + .locator('p.bn-mt-suggestion-menu-item-title') + .getByText('Accept') + .click(); + + await expect(editor.getByText('Bonjour le monde')).toBeVisible(); + + // Check Suggestion menu + await page.locator('.bn-block-outer').last().fill('/'); + await expect(page.getByText('Write with AI')).toBeVisible(); + }); + test('it checks the AI buttons', async ({ page, browserName }) => { await page.route(/.*\/ai-translate\//, async (route) => { const request = route.request(); diff --git a/src/frontend/apps/impress/package.json b/src/frontend/apps/impress/package.json index cfb372cb..085a7317 100644 --- a/src/frontend/apps/impress/package.json +++ b/src/frontend/apps/impress/package.json @@ -19,10 +19,12 @@ }, "dependencies": { "@ag-media/react-pdf-table": "2.0.3", + "@ai-sdk/openai": "1.3.22", "@blocknote/code-block": "0.45.0", "@blocknote/core": "0.45.0", "@blocknote/mantine": "0.45.0", "@blocknote/react": "0.45.0", + "@blocknote/xl-ai": "0.45.0", "@blocknote/xl-docx-exporter": "0.45.0", "@blocknote/xl-multi-column": "0.45.0", "@blocknote/xl-odt-exporter": "0.45.0", @@ -44,6 +46,7 @@ "@sentry/nextjs": "10.32.1", "@tanstack/react-query": "5.90.16", "@tiptap/extensions": "*", + "ai": "4.3.16", "canvg": "4.0.3", "clsx": "2.1.1", "cmdk": "1.1.1", @@ -72,6 +75,7 @@ "uuid": "13.0.0", "y-protocols": "1.0.7", "yjs": "*", + "zod": "3.25.28", "zustand": "5.0.9" }, "devDependencies": { diff --git a/src/frontend/apps/impress/src/features/docs/doc-editor/assets/IconAI.svg b/src/frontend/apps/impress/src/features/docs/doc-editor/assets/IconAI.svg new file mode 100644 index 00000000..8436d004 --- /dev/null +++ b/src/frontend/apps/impress/src/features/docs/doc-editor/assets/IconAI.svg @@ -0,0 +1,6 @@ + + + diff --git a/src/frontend/apps/impress/src/features/docs/doc-editor/components/AI/AIUI.tsx b/src/frontend/apps/impress/src/features/docs/doc-editor/components/AI/AIUI.tsx new file mode 100644 index 00000000..cea52921 --- /dev/null +++ b/src/frontend/apps/impress/src/features/docs/doc-editor/components/AI/AIUI.tsx @@ -0,0 +1,135 @@ +import { useBlockNoteEditor, useComponentsContext } from '@blocknote/react'; +import { + AIMenu as AIMenuDefault, + getAIExtension, + getDefaultAIMenuItems, +} from '@blocknote/xl-ai'; +import { useTranslation } from 'react-i18next'; +import { css } from 'styled-components'; + +import { Box, Text } from '@/components'; +import { useCunninghamTheme } from '@/cunningham'; + +import IconAI from '../../assets/IconAI.svg'; +import { + DocsBlockNoteEditor, + DocsBlockSchema, + DocsInlineContentSchema, + DocsStyleSchema, +} from '../../types'; + +export function AIMenu() { + return ( + { + if (aiResponseStatus === 'user-input') { + if (editor.getSelection()) { + const aiMenuItems = getDefaultAIMenuItems( + editor, + aiResponseStatus, + ).filter((item) => ['simplify'].indexOf(item.key) === -1); + + return aiMenuItems; + } else { + const aiMenuItems = getDefaultAIMenuItems( + editor, + aiResponseStatus, + ).filter( + (item) => + ['action_items', 'write_anything'].indexOf(item.key) === -1, + ); + + return aiMenuItems; + } + } + + return getDefaultAIMenuItems(editor, aiResponseStatus); + }} + /> + ); +} + +export const AIToolbarButton = () => { + const Components = useComponentsContext(); + const { t } = useTranslation(); + const { spacingsTokens, colorsTokens } = useCunninghamTheme(); + const editor = useBlockNoteEditor< + DocsBlockSchema, + DocsInlineContentSchema, + DocsStyleSchema + >(); + + if (!editor.isEditable || !Components) { + return null; + } + + const onClick = () => { + const aiExtension = getAIExtension(editor); + editor.formattingToolbar.closeMenu(); + const selection = editor.getSelection(); + if (!selection) { + throw new Error('No selection'); + } + + const position = selection.blocks[selection.blocks.length - 1].id; + aiExtension.openAIMenuAtBlock(position); + }; + + return ( + button.mantine-Button-root { + padding-inline: ${spacingsTokens['2xs']}; + transition: all 0.1s ease-in; + &:hover, + &:hover { + background-color: ${colorsTokens['greyscale-050']}; + } + &:hover .--docs--icon-bg { + background-color: #5858e1; + border: 1px solid #8484f5; + color: #ffffff; + } + } + `} + $direction="row" + className="--docs--ai-toolbar-button" + > + + + + + + {t('Ask AI')} + + + + + ); +}; diff --git a/src/frontend/apps/impress/src/features/docs/doc-editor/components/AI/index.ts b/src/frontend/apps/impress/src/features/docs/doc-editor/components/AI/index.ts new file mode 100644 index 00000000..fe820022 --- /dev/null +++ b/src/frontend/apps/impress/src/features/docs/doc-editor/components/AI/index.ts @@ -0,0 +1,2 @@ +export * from './AIUI'; +export * from './useAI'; diff --git a/src/frontend/apps/impress/src/features/docs/doc-editor/components/AI/useAI.tsx b/src/frontend/apps/impress/src/features/docs/doc-editor/components/AI/useAI.tsx new file mode 100644 index 00000000..b0f791e2 --- /dev/null +++ b/src/frontend/apps/impress/src/features/docs/doc-editor/components/AI/useAI.tsx @@ -0,0 +1,47 @@ +import { createOpenAI } from '@ai-sdk/openai'; +import { createAIExtension, createBlockNoteAIClient } from '@blocknote/xl-ai'; +import { useMemo } from 'react'; + +import { fetchAPI } from '@/api'; +import { Doc } from '@/docs/doc-management'; + +const client = createBlockNoteAIClient({ + baseURL: ``, + apiKey: '', +}); + +/** + * Custom implementation of the PromptBuilder that allows for using predefined prompts. + * + * This extends the default HTML promptBuilder from BlockNote to support custom prompt templates. + * Custom prompts can be invoked using the pattern !promptName in the AI input field. + */ +export const useAI = (docId: Doc['id']) => { + return useMemo(() => { + const openai = createOpenAI({ + ...client.getProviderSettings('openai'), + fetch: (input, init) => { + // Create a new headers object without the Authorization header + const headers = new Headers(init?.headers); + headers.delete('Authorization'); + + return fetchAPI(`documents/${docId}/ai-proxy/`, { + ...init, + headers, + }); + }, + }); + const model = openai.chat('neuralmagic/Meta-Llama-3.1-70B-Instruct-FP8'); + + const extension = createAIExtension({ + stream: false, + model, + agentCursor: { + name: 'Albert', + color: '#8bc6ff', + }, + }); + + return extension; + }, [docId]); +}; diff --git a/src/frontend/apps/impress/src/features/docs/doc-editor/components/BlockNoteEditor.tsx b/src/frontend/apps/impress/src/features/docs/doc-editor/components/BlockNoteEditor.tsx index e0be2f28..ba866b07 100644 --- a/src/frontend/apps/impress/src/features/docs/doc-editor/components/BlockNoteEditor.tsx +++ b/src/frontend/apps/impress/src/features/docs/doc-editor/components/BlockNoteEditor.tsx @@ -12,6 +12,9 @@ import * as locales from '@blocknote/core/locales'; import { BlockNoteView } from '@blocknote/mantine'; import '@blocknote/mantine/style.css'; import { useCreateBlockNote } from '@blocknote/react'; +import { AIMenuController } from '@blocknote/xl-ai'; +import { en as aiEn } from '@blocknote/xl-ai/locales'; +import '@blocknote/xl-ai/style.css'; import { HocuspocusProvider } from '@hocuspocus/provider'; import { useEffect, useMemo, useRef } from 'react'; import { useTranslation } from 'react-i18next'; @@ -36,6 +39,7 @@ import { cssEditor } from '../styles'; import { DocsBlockNoteEditor } from '../types'; import { randomColor } from '../utils'; +import { AIMenu, useAI } from './AI'; import { BlockNoteSuggestionMenu } from './BlockNoteSuggestionMenu'; import { BlockNoteToolbar } from './BlockNoteToolBar/BlockNoteToolbar'; import { cssComments, useComments } from './comments/'; @@ -99,6 +103,7 @@ export const BlockNoteEditor = ({ doc, provider }: BlockNoteEditorProps) => { } const { uploadFile, errorAttachment } = useUploadFile(doc.id); + const aiExtension = useAI(doc.id); const collabName = user?.full_name || user?.email; const cursorName = collabName || t('Anonymous'); @@ -168,6 +173,7 @@ export const BlockNoteEditor = ({ doc, provider }: BlockNoteEditorProps) => { ...(multiColumnLocales && { multi_column: multiColumnLocales[lang as keyof typeof multiColumnLocales], + ai: aiEn, }), }, pasteHandler: ({ event, defaultPasteHandler }) => { @@ -190,7 +196,10 @@ export const BlockNoteEditor = ({ doc, provider }: BlockNoteEditorProps) => { return defaultPasteHandler(); }, - extensions: [CommentsExtension({ threadStore, resolveUsers })], + extensions: [ + CommentsExtension({ threadStore, resolveUsers }), + aiExtension, + ], tables: { splitCells: true, cellBackgroundColor: true, @@ -243,6 +252,7 @@ export const BlockNoteEditor = ({ doc, provider }: BlockNoteEditorProps) => { comments={showComments} aria-label={t('Document editor')} > + diff --git a/src/frontend/apps/impress/src/features/docs/doc-editor/components/BlockNoteSuggestionMenu.tsx b/src/frontend/apps/impress/src/features/docs/doc-editor/components/BlockNoteSuggestionMenu.tsx index e07d15bc..35d60549 100644 --- a/src/frontend/apps/impress/src/features/docs/doc-editor/components/BlockNoteSuggestionMenu.tsx +++ b/src/frontend/apps/impress/src/features/docs/doc-editor/components/BlockNoteSuggestionMenu.tsx @@ -8,9 +8,12 @@ import { useBlockNoteEditor, useDictionary, } from '@blocknote/react'; +import { getAISlashMenuItems } from '@blocknote/xl-ai'; import React, { useMemo } from 'react'; import { useTranslation } from 'react-i18next'; +import { useConfig } from '@/core'; + import { DocsBlockSchema, DocsInlineContentSchema, @@ -39,6 +42,7 @@ export const BlockNoteSuggestionMenu = () => { const fileBlocksName = dictionaryDate.slash_menu.file.group; const getInterlinkingMenuItems = useGetInterlinkingMenuItems(); + const { data: conf } = useConfig(); const getSlashMenuItems = useMemo(() => { // We insert it after the "Code Block" item to have the interlinking block displayed after the basic blocks @@ -50,6 +54,7 @@ export const BlockNoteSuggestionMenu = () => { getMultiColumnSlashMenuItems?.(editor) || [], getPdfReactSlashMenuItems(editor, t, fileBlocksName), getCalloutReactSlashMenuItems(editor, t, basicBlocksName), + conf?.AI_FEATURE_ENABLED ? getAISlashMenuItems(editor) : [], ); const index = combinedMenu.findIndex( @@ -66,7 +71,14 @@ export const BlockNoteSuggestionMenu = () => { return async (query: string) => Promise.resolve(filterSuggestionItems(newSlashMenuItems, query)); - }, [basicBlocksName, editor, getInterlinkingMenuItems, t, fileBlocksName]); + }, [ + editor, + t, + fileBlocksName, + basicBlocksName, + conf?.AI_FEATURE_ENABLED, + getInterlinkingMenuItems, + ]); return ( { const formattingToolbar = useCallback(() => { return ( + {conf?.AI_FEATURE_ENABLED && } + {toolbarItems} diff --git a/src/frontend/apps/impress/src/features/docs/doc-editor/hook/useSaveDoc.tsx b/src/frontend/apps/impress/src/features/docs/doc-editor/hook/useSaveDoc.tsx index a5d1d585..fdbf3357 100644 --- a/src/frontend/apps/impress/src/features/docs/doc-editor/hook/useSaveDoc.tsx +++ b/src/frontend/apps/impress/src/features/docs/doc-editor/hook/useSaveDoc.tsx @@ -35,7 +35,13 @@ export const useSaveDoc = ( _updatedDoc: Y.Doc, transaction: Y.Transaction, ) => { - setIsLocalChange(transaction.local); + /** + * When the AI edit the doc transaction.local is false, + * so we check if the origin is null to know if the change + * is local or not. + * TODO: see if we can get the local changes from the AI + */ + setIsLocalChange(transaction.local || transaction.origin === null); }; yDoc.on('update', onUpdate); diff --git a/src/frontend/apps/impress/src/features/docs/doc-editor/styles.tsx b/src/frontend/apps/impress/src/features/docs/doc-editor/styles.tsx index 6c9c1645..71afc420 100644 --- a/src/frontend/apps/impress/src/features/docs/doc-editor/styles.tsx +++ b/src/frontend/apps/impress/src/features/docs/doc-editor/styles.tsx @@ -1,6 +1,11 @@ import { css } from 'styled-components'; export const cssEditor = css` + .mantine-Menu-itemLabel, + .mantine-Button-label { + font-family: var(--c--components--button--font-family); + } + &, & > .bn-container, & .ProseMirror { @@ -148,6 +153,19 @@ export const cssEditor = css` font-style: italic; } + /** + * AI + */ + ins, + [data-type='modification'] { + background: var(--c--theme--colors--primary-100); + border-bottom: 2px solid var(--c--theme--colors--primary-300); + color: var(--c--theme--colors--primary-700); + } + [data-show-selection] { + background-color: var(--c--theme--colors--primary-300); + } + /** * Divider */