✨(frontend) integrate new Blocknote AI feature

We integrate the new Blocknote AI feature into Docs, enhancing the document editing experience with AI capabilities.
2026-01-27 17:53:17 +01:00
parent cc7ed88498
commit 08b04dea90
13 changed files with 346 additions and 6 deletions
@@ -12,6 +12,7 @@ and this project adheres to
 - ✨ Import of documents #1609
 - 🚨(CI) gives warning if theme not updated #1811
 - 🔧(project) add DJANGO_EMAIL_URL_APP environment variable #1825
+- ✨(frontend) integrate new Blocknote AI feature #1016

 ### Changed

@@ -93,9 +93,7 @@ test.describe('Config', () => {
    expect(
      await page.locator('button[data-test="convertMarkdown"]').count(),
    ).toBe(1);
-    expect(await page.locator('button[data-test="ai-actions"]').count()).toBe(
-      0,
-    );
+    await expect(page.getByRole('button', { name: 'Ask AI' })).toBeHidden();
  });

  test('it checks that Crisp is trying to init from config endpoint', async ({
@@ -389,6 +389,104 @@ test.describe('Doc Editor', () => {
    await expect(image).toHaveAttribute('aria-hidden', 'true');
  });

+  test('it checks the AI feature', async ({ page, browserName }) => {
+    await page.route(/.*\/ai-proxy\//, async (route) => {
+      const request = route.request();
+      if (request.method().includes('POST')) {
+        await route.fulfill({
+          json: {
+            id: 'chatcmpl-b1e7a9e456ca41f78fec130d552a6bf5',
+            choices: [
+              {
+                finish_reason: 'stop',
+                index: 0,
+                logprobs: null,
+                message: {
+                  content: '',
+                  refusal: null,
+                  role: 'assistant',
+                  annotations: null,
+                  audio: null,
+                  function_call: null,
+                  tool_calls: [
+                    {
+                      id: 'chatcmpl-tool-2e3567dfecf94a4c85e27a3528337718',
+                      function: {
+                        arguments:
+                          '{"operations": [{"type": "update", "id": "initialBlockId$", "block": "<p>Bonjour le monde</p>"}]}',
+                        name: 'json',
+                      },
+                      type: 'function',
+                    },
+                  ],
+                  reasoning_content: null,
+                },
+                stop_reason: null,
+              },
+            ],
+            created: 1749549477,
+            model: 'neuralmagic/Meta-Llama-3.1-70B-Instruct-FP8',
+            object: 'chat.completion',
+            service_tier: null,
+            system_fingerprint: null,
+            usage: {
+              completion_tokens: 0,
+              prompt_tokens: 204,
+              total_tokens: 204,
+              completion_tokens_details: null,
+              prompt_tokens_details: null,
+              details: [
+                {
+                  id: 'chatcmpl-b1e7a9e456ca41f78fec130d552a6bf5',
+                  model: 'neuralmagic/Meta-Llama-3.1-70B-Instruct-FP8',
+                  prompt_tokens: 204,
+                  completion_tokens: 0,
+                  total_tokens: 204,
+                },
+              ],
+            },
+            prompt_logprobs: null,
+          },
+        });
+      } else {
+        await route.continue();
+      }
+    });
+
+    await createDoc(page, 'doc-ai', browserName, 1);
+
+    await page.locator('.bn-block-outer').last().fill('Hello World');
+
+    const editor = page.locator('.ProseMirror');
+    await editor.getByText('Hello World').selectText();
+
+    // Check from toolbar
+    await page.getByRole('button', { name: 'Ask AI' }).click();
+
+    await expect(
+      page.getByRole('option', { name: 'Improve Writing' }),
+    ).toBeVisible();
+    await expect(
+      page.getByRole('option', { name: 'Fix Spelling' }),
+    ).toBeVisible();
+    await expect(page.getByRole('option', { name: 'Translate' })).toBeVisible();
+
+    await page.getByRole('option', { name: 'Translate' }).click();
+    await page.getByPlaceholder('Ask AI anything…').fill('French');
+    await page.getByPlaceholder('Ask AI anything…').press('Enter');
+    await expect(editor.getByText('Docs AI')).toBeVisible();
+    await page
+      .locator('p.bn-mt-suggestion-menu-item-title')
+      .getByText('Accept')
+      .click();
+
+    await expect(editor.getByText('Bonjour le monde')).toBeVisible();
+
+    // Check Suggestion menu
+    await page.locator('.bn-block-outer').last().fill('/');
+    await expect(page.getByText('Write with AI')).toBeVisible();
+  });
+
  test('it checks the AI buttons', async ({ page, browserName }) => {
    await page.route(/.*\/ai-translate\//, async (route) => {
      const request = route.request();
@@ -19,10 +19,12 @@
  },
  "dependencies": {
    "@ag-media/react-pdf-table": "2.0.3",
+    "@ai-sdk/openai": "1.3.22",
    "@blocknote/code-block": "0.45.0",
    "@blocknote/core": "0.45.0",
    "@blocknote/mantine": "0.45.0",
    "@blocknote/react": "0.45.0",
+    "@blocknote/xl-ai": "0.45.0",
    "@blocknote/xl-docx-exporter": "0.45.0",
    "@blocknote/xl-multi-column": "0.45.0",
    "@blocknote/xl-odt-exporter": "0.45.0",
@@ -44,6 +46,7 @@
    "@sentry/nextjs": "10.32.1",
    "@tanstack/react-query": "5.90.16",
    "@tiptap/extensions": "*",
+    "ai": "4.3.16",
    "canvg": "4.0.3",
    "clsx": "2.1.1",
    "cmdk": "1.1.1",
@@ -72,6 +75,7 @@
    "uuid": "13.0.0",
    "y-protocols": "1.0.7",
    "yjs": "*",
+    "zod": "3.25.28",
    "zustand": "5.0.9"
  },
  "devDependencies": {
@@ -0,0 +1,6 @@
+<svg viewBox="0 0 10 10" fill="none" xmlns="http://www.w3.org/2000/svg">
+  <path
+    d="M5.89904 7.9473C6.0847 7.9473 6.19816 7.84673 6.23941 7.6456C6.31677 7.17114 6.39671 6.76887 6.47922 6.43881C6.56174 6.10359 6.66746 5.82768 6.79639 5.61108C6.92532 5.38932 7.09551 5.21139 7.30696 5.0773C7.5184 4.93806 7.78916 4.82718 8.11922 4.74466C8.44928 4.66215 8.85928 4.58737 9.34922 4.52033C9.56066 4.49454 9.66639 4.3785 9.66639 4.17221C9.66639 3.98139 9.56066 3.87051 9.34922 3.83957C8.8696 3.77768 8.46475 3.70548 8.13469 3.62297C7.80979 3.53529 7.53903 3.42441 7.32243 3.29033C7.11098 3.15624 6.93822 2.98089 6.80413 2.76429C6.6752 2.54253 6.5669 2.26662 6.47922 1.93656C6.39671 1.60649 6.31677 1.20165 6.23941 0.722032C6.19816 0.515743 6.0847 0.412598 5.89904 0.412598C5.70306 0.412598 5.58702 0.515743 5.55092 0.722032C5.47872 1.1965 5.40136 1.60134 5.31885 1.93656C5.23633 2.26662 5.12803 2.53995 4.99394 2.75655C4.86501 2.97316 4.69483 3.15108 4.48338 3.29033C4.27193 3.42441 4.00118 3.53272 3.67112 3.61523C3.34106 3.69775 2.93364 3.77253 2.44886 3.83957C2.23741 3.87051 2.13169 3.98139 2.13169 4.17221C2.13169 4.36819 2.23741 4.48422 2.44886 4.52033C3.04709 4.60284 3.52929 4.70083 3.89546 4.81429C4.26162 4.92774 4.54784 5.0902 4.75413 5.30164C4.96042 5.51309 5.1203 5.80705 5.23376 6.18353C5.35237 6.55485 5.45809 7.04221 5.55092 7.6456C5.59218 7.84673 5.70822 7.9473 5.89904 7.9473ZM2.53395 9.27786C2.6732 9.27786 2.75829 9.20824 2.78923 9.06899C2.83565 8.77503 2.87691 8.54296 2.91301 8.37277C2.95426 8.20774 3.01357 8.08138 3.09093 7.99371C3.17345 7.90604 3.30238 7.839 3.47772 7.79258C3.65307 7.74617 3.90061 7.69459 4.22036 7.63786C4.35961 7.61208 4.42923 7.52956 4.42923 7.39032C4.42923 7.25623 4.35961 7.17629 4.22036 7.15051C3.90061 7.09894 3.65307 7.05252 3.47772 7.01126C3.30753 6.96485 3.1786 6.90038 3.09093 6.81787C3.00841 6.73019 2.94911 6.60126 2.91301 6.43107C2.87691 6.26089 2.83565 6.02623 2.78923 5.72711C2.75313 5.57756 2.66804 5.50278 2.53395 5.50278C2.40502 5.50278 2.3225 5.57756 2.2864 5.72711C2.23483 6.02108 2.19099 6.25057 2.15489 6.4156C2.11879 6.58063 2.05949 6.70699 1.97697 6.79466C1.89961 6.87717 1.77326 6.94164 1.59791 6.98805C1.42773 7.03447 1.18276 7.08862 0.863011 7.15051C0.713451 7.17629 0.638672 7.25623 0.638672 7.39032C0.638672 7.52956 0.721187 7.61208 0.886218 7.63786C1.19565 7.68944 1.43546 7.73843 1.60565 7.78485C1.77584 7.8261 1.89961 7.89057 1.97697 7.97824C2.05949 8.06592 2.11879 8.19227 2.15489 8.3573C2.19099 8.52749 2.23483 8.75956 2.2864 9.05352C2.32766 9.20308 2.41018 9.27786 2.53395 9.27786Z"
+    fill="currentColor"
+  />
+</svg>
@@ -0,0 +1,135 @@
+import { useBlockNoteEditor, useComponentsContext } from '@blocknote/react';
+import {
+  AIMenu as AIMenuDefault,
+  getAIExtension,
+  getDefaultAIMenuItems,
+} from '@blocknote/xl-ai';
+import { useTranslation } from 'react-i18next';
+import { css } from 'styled-components';
+
+import { Box, Text } from '@/components';
+import { useCunninghamTheme } from '@/cunningham';
+
+import IconAI from '../../assets/IconAI.svg';
+import {
+  DocsBlockNoteEditor,
+  DocsBlockSchema,
+  DocsInlineContentSchema,
+  DocsStyleSchema,
+} from '../../types';
+
+export function AIMenu() {
+  return (
+    <AIMenuDefault
+      items={(editor: DocsBlockNoteEditor, aiResponseStatus) => {
+        if (aiResponseStatus === 'user-input') {
+          if (editor.getSelection()) {
+            const aiMenuItems = getDefaultAIMenuItems(
+              editor,
+              aiResponseStatus,
+            ).filter((item) => ['simplify'].indexOf(item.key) === -1);
+
+            return aiMenuItems;
+          } else {
+            const aiMenuItems = getDefaultAIMenuItems(
+              editor,
+              aiResponseStatus,
+            ).filter(
+              (item) =>
+                ['action_items', 'write_anything'].indexOf(item.key) === -1,
+            );
+
+            return aiMenuItems;
+          }
+        }
+
+        return getDefaultAIMenuItems(editor, aiResponseStatus);
+      }}
+    />
+  );
+}
+
+export const AIToolbarButton = () => {
+  const Components = useComponentsContext();
+  const { t } = useTranslation();
+  const { spacingsTokens, colorsTokens } = useCunninghamTheme();
+  const editor = useBlockNoteEditor<
+    DocsBlockSchema,
+    DocsInlineContentSchema,
+    DocsStyleSchema
+  >();
+
+  if (!editor.isEditable || !Components) {
+    return null;
+  }
+
+  const onClick = () => {
+    const aiExtension = getAIExtension(editor);
+    editor.formattingToolbar.closeMenu();
+    const selection = editor.getSelection();
+    if (!selection) {
+      throw new Error('No selection');
+    }
+
+    const position = selection.blocks[selection.blocks.length - 1].id;
+    aiExtension.openAIMenuAtBlock(position);
+  };
+
+  return (
+    <Box
+      $css={css`
+        & > button.mantine-Button-root {
+          padding-inline: ${spacingsTokens['2xs']};
+          transition: all 0.1s ease-in;
+          &:hover,
+          &:hover {
+            background-color: ${colorsTokens['greyscale-050']};
+          }
+          &:hover .--docs--icon-bg {
+            background-color: #5858e1;
+            border: 1px solid #8484f5;
+            color: #ffffff;
+          }
+        }
+      `}
+      $direction="row"
+      className="--docs--ai-toolbar-button"
+    >
+      <Components.Generic.Toolbar.Button
+        className="bn-button"
+        onClick={onClick}
+      >
+        <Box
+          $direction="row"
+          $align="center"
+          $gap={spacingsTokens['xs']}
+          $padding={{ right: '2xs' }}
+        >
+          <Text
+            className="--docs--icon-bg"
+            $theme="greyscale"
+            $variation="600"
+            $css={css`
+              border: 1px solid var(--c--theme--colors--greyscale-100);
+              transition: all 0.1s ease-in;
+            `}
+            $radius="100%"
+            $padding="0.15rem"
+          >
+            <IconAI width="16px" />
+          </Text>
+          {t('Ask AI')}
+        </Box>
+      </Components.Generic.Toolbar.Button>
+      <Box
+        $background={colorsTokens['greyscale-100']}
+        $width="1px"
+        $height="70%"
+        $margin={{ left: '2px' }}
+        $css={css`
+          align-self: center;
+        `}
+      />
+    </Box>
+  );
+};
@@ -0,0 +1,2 @@
+export * from './AIUI';
+export * from './useAI';
@@ -0,0 +1,47 @@
+import { createOpenAI } from '@ai-sdk/openai';
+import { createAIExtension, createBlockNoteAIClient } from '@blocknote/xl-ai';
+import { useMemo } from 'react';
+
+import { fetchAPI } from '@/api';
+import { Doc } from '@/docs/doc-management';
+
+const client = createBlockNoteAIClient({
+  baseURL: ``,
+  apiKey: '',
+});
+
+/**
+ * Custom implementation of the PromptBuilder that allows for using predefined prompts.
+ *
+ * This extends the default HTML promptBuilder from BlockNote to support custom prompt templates.
+ * Custom prompts can be invoked using the pattern !promptName in the AI input field.
+ */
+export const useAI = (docId: Doc['id']) => {
+  return useMemo(() => {
+    const openai = createOpenAI({
+      ...client.getProviderSettings('openai'),
+      fetch: (input, init) => {
+        // Create a new headers object without the Authorization header
+        const headers = new Headers(init?.headers);
+        headers.delete('Authorization');
+
+        return fetchAPI(`documents/${docId}/ai-proxy/`, {
+          ...init,
+          headers,
+        });
+      },
+    });
+    const model = openai.chat('neuralmagic/Meta-Llama-3.1-70B-Instruct-FP8');
+
+    const extension = createAIExtension({
+      stream: false,
+      model,
+      agentCursor: {
+        name: 'Albert',
+        color: '#8bc6ff',
+      },
+    });
+
+    return extension;
+  }, [docId]);
+};
@@ -12,6 +12,9 @@ import * as locales from '@blocknote/core/locales';
 import { BlockNoteView } from '@blocknote/mantine';
 import '@blocknote/mantine/style.css';
 import { useCreateBlockNote } from '@blocknote/react';
+import { AIMenuController } from '@blocknote/xl-ai';
+import { en as aiEn } from '@blocknote/xl-ai/locales';
+import '@blocknote/xl-ai/style.css';
 import { HocuspocusProvider } from '@hocuspocus/provider';
 import { useEffect, useMemo, useRef } from 'react';
 import { useTranslation } from 'react-i18next';
@@ -36,6 +39,7 @@ import { cssEditor } from '../styles';
 import { DocsBlockNoteEditor } from '../types';
 import { randomColor } from '../utils';

+import { AIMenu, useAI } from './AI';
 import { BlockNoteSuggestionMenu } from './BlockNoteSuggestionMenu';
 import { BlockNoteToolbar } from './BlockNoteToolBar/BlockNoteToolbar';
 import { cssComments, useComments } from './comments/';
@@ -99,6 +103,7 @@ export const BlockNoteEditor = ({ doc, provider }: BlockNoteEditorProps) => {
  }

  const { uploadFile, errorAttachment } = useUploadFile(doc.id);
+  const aiExtension = useAI(doc.id);

  const collabName = user?.full_name || user?.email;
  const cursorName = collabName || t('Anonymous');
@@ -168,6 +173,7 @@ export const BlockNoteEditor = ({ doc, provider }: BlockNoteEditorProps) => {
        ...(multiColumnLocales && {
          multi_column:
            multiColumnLocales[lang as keyof typeof multiColumnLocales],
+          ai: aiEn,
        }),
      },
      pasteHandler: ({ event, defaultPasteHandler }) => {
@@ -190,7 +196,10 @@ export const BlockNoteEditor = ({ doc, provider }: BlockNoteEditorProps) => {

        return defaultPasteHandler();
      },
-      extensions: [CommentsExtension({ threadStore, resolveUsers })],
+      extensions: [
+        CommentsExtension({ threadStore, resolveUsers }),
+        aiExtension,
+      ],
      tables: {
        splitCells: true,
        cellBackgroundColor: true,
@@ -243,6 +252,7 @@ export const BlockNoteEditor = ({ doc, provider }: BlockNoteEditorProps) => {
        comments={showComments}
        aria-label={t('Document editor')}
      >
+        <AIMenuController aiMenu={AIMenu} />
        <BlockNoteSuggestionMenu />
        <BlockNoteToolbar />
      </BlockNoteView>
@@ -8,9 +8,12 @@ import {
  useBlockNoteEditor,
  useDictionary,
 } from '@blocknote/react';
+import { getAISlashMenuItems } from '@blocknote/xl-ai';
 import React, { useMemo } from 'react';
 import { useTranslation } from 'react-i18next';

+import { useConfig } from '@/core';
+
 import {
  DocsBlockSchema,
  DocsInlineContentSchema,
@@ -39,6 +42,7 @@ export const BlockNoteSuggestionMenu = () => {
  const fileBlocksName = dictionaryDate.slash_menu.file.group;

  const getInterlinkingMenuItems = useGetInterlinkingMenuItems();
+  const { data: conf } = useConfig();

  const getSlashMenuItems = useMemo(() => {
    // We insert it after the "Code Block" item to have the interlinking block displayed after the basic blocks
@@ -50,6 +54,7 @@ export const BlockNoteSuggestionMenu = () => {
      getMultiColumnSlashMenuItems?.(editor) || [],
      getPdfReactSlashMenuItems(editor, t, fileBlocksName),
      getCalloutReactSlashMenuItems(editor, t, basicBlocksName),
+      conf?.AI_FEATURE_ENABLED ? getAISlashMenuItems(editor) : [],
    );

    const index = combinedMenu.findIndex(
@@ -66,7 +71,14 @@ export const BlockNoteSuggestionMenu = () => {

    return async (query: string) =>
      Promise.resolve(filterSuggestionItems(newSlashMenuItems, query));
-  }, [basicBlocksName, editor, getInterlinkingMenuItems, t, fileBlocksName]);
+  }, [
+    editor,
+    t,
+    fileBlocksName,
+    basicBlocksName,
+    conf?.AI_FEATURE_ENABLED,
+    getInterlinkingMenuItems,
+  ]);

  return (
    <SuggestionMenuController
@@ -10,6 +10,7 @@ import { useTranslation } from 'react-i18next';

 import { useConfig } from '@/core/config/api';

+import { AIToolbarButton } from '../AI/AIToolbarButton';
 import { CommentToolbarButton } from '../comments/CommentToolbarButton';
 import { getCalloutFormattingToolbarItems } from '../custom-blocks';

@@ -69,6 +70,8 @@ export const BlockNoteToolbar = () => {
  const formattingToolbar = useCallback(() => {
    return (
      <FormattingToolbar>
+        {conf?.AI_FEATURE_ENABLED && <AIToolbarButton />}
+
        <CommentToolbarButton />

        {toolbarItems}
@@ -35,7 +35,13 @@ export const useSaveDoc = (
      _updatedDoc: Y.Doc,
      transaction: Y.Transaction,
    ) => {
-      setIsLocalChange(transaction.local);
+      /**
+       * When the AI edit the doc transaction.local is false,
+       * so we check if the origin is null to know if the change
+       * is local or not.
+       * TODO: see if we can get the local changes from the AI
+       */
+      setIsLocalChange(transaction.local || transaction.origin === null);
    };

    yDoc.on('update', onUpdate);
@@ -1,6 +1,11 @@
 import { css } from 'styled-components';

 export const cssEditor = css`
+  .mantine-Menu-itemLabel,
+  .mantine-Button-label {
+    font-family: var(--c--components--button--font-family);
+  }
+
  &,
  & > .bn-container,
  & .ProseMirror {
@@ -148,6 +153,19 @@ export const cssEditor = css`
    font-style: italic;
  }

+  /**
+    * AI
+    */
+  ins,
+  [data-type='modification'] {
+    background: var(--c--theme--colors--primary-100);
+    border-bottom: 2px solid var(--c--theme--colors--primary-300);
+    color: var(--c--theme--colors--primary-700);
+  }
+  [data-show-selection] {
+    background-color: var(--c--theme--colors--primary-300);
+  }
+
  /**
  * Divider
  */