From 33baf2e8f866bf0ff6297e541622f27009347195 Mon Sep 17 00:00:00 2001 From: yousefed Date: Thu, 22 May 2025 13:35:51 +0200 Subject: [PATCH 1/2] docs: add attention to new AI features --- docs/components/Navigation.tsx | 11 ++++++++++- docs/components/navigation.css | 13 +++++++++++++ docs/pages/_meta.json | 5 +++++ 3 files changed, 28 insertions(+), 1 deletion(-) create mode 100644 docs/components/navigation.css diff --git a/docs/components/Navigation.tsx b/docs/components/Navigation.tsx index be25a4b387..7d900cb8c7 100644 --- a/docs/components/Navigation.tsx +++ b/docs/components/Navigation.tsx @@ -3,9 +3,18 @@ /* eslint-disable @typescript-eslint/no-unsafe-assignment -- Lots of Nextra magic. */ import { Navbar } from "nextra-theme-docs"; +import "./navigation.css"; export function Navigation(props: any) { // items last to override the default // return
hello
; - return ; + return ( + <> +
+ 🚀 BlockNote AI is here!{" "} + Access the early preview. +
+ + + ); } diff --git a/docs/components/navigation.css b/docs/components/navigation.css new file mode 100644 index 0000000000..d5c9a65566 --- /dev/null +++ b/docs/components/navigation.css @@ -0,0 +1,13 @@ +.top-banner { + background-color: #fef6d5; + color: #000; + text-align: center; + padding: 0.2em 0; + font-size: 0.8em; + font-style: italic; + border-bottom: 1px solid #e5e7eb; +} + +.top-banner a { + text-decoration: underline; +} diff --git a/docs/pages/_meta.json b/docs/pages/_meta.json index 96782e5b0b..f98b166842 100644 --- a/docs/pages/_meta.json +++ b/docs/pages/_meta.json @@ -16,6 +16,11 @@ "title": "Docs", "display": "children" }, + "ai": { + "title": "AI", + "href": "/docs/ai", + "route": "/needs-fake-route-otherwise-menu-shows-bold" + }, "examples": { "title": "Examples", "display": "children" From ffc887a44a155db34b07fa469195219b3c64ffc7 Mon Sep 17 00:00:00 2001 From: yousefed Date: Thu, 29 May 2025 16:32:34 +0200 Subject: [PATCH 2/2] text fading / ai tools --- .../xl-ai/src/plugins/AgentCursorPlugin.ts | 19 ++++++- packages/xl-ai/src/prosemirror/agent.ts | 1 + packages/xl-ai/src/streamTool/asTool.ts | 47 +++++++++++------ .../src/streamTool/callLLMWithStreamTools.ts | 41 ++++++++------- packages/xl-ai/src/streamTool/preprocess.ts | 4 +- .../src/streamTool/toValidatedOperations.ts | 50 +++++++++++-------- packages/xl-ai/src/style.css | 37 +++++++++++++- 7 files changed, 139 insertions(+), 60 deletions(-) diff --git a/packages/xl-ai/src/plugins/AgentCursorPlugin.ts b/packages/xl-ai/src/plugins/AgentCursorPlugin.ts index f2492e1a08..d2ffd6ed70 100644 --- a/packages/xl-ai/src/plugins/AgentCursorPlugin.ts +++ b/packages/xl-ai/src/plugins/AgentCursorPlugin.ts @@ -4,6 +4,7 @@ import { defaultSelectionBuilder } from "y-prosemirror"; type AgentCursorState = { selection: { anchor: number; head: number } | undefined; + charPositions: number[]; }; const PLUGIN_KEY = new PluginKey(`blocknote-agent-cursor`); @@ -23,19 +24,26 @@ export function createAgentCursorPlugin(agentCursor: { init: () => { return { selection: undefined, + charPositions: [], }; }, - apply: (tr, _oldState) => { + apply: (tr, oldState) => { const meta = tr.getMeta("aiAgent"); if (!meta) { return { selection: undefined, + charPositions: [], }; } + const prevCharPositions = oldState.charPositions; + if (meta.selection.inserted) { + prevCharPositions.push(meta.selection.anchor); + } return { selection: meta.selection, + charPositions: prevCharPositions, }; }, }, @@ -43,7 +51,7 @@ export function createAgentCursorPlugin(agentCursor: { decorations: (state) => { const { doc } = state; - const { selection } = PLUGIN_KEY.getState(state)!; + const { selection, charPositions } = PLUGIN_KEY.getState(state)!; const decs = []; @@ -68,6 +76,13 @@ export function createAgentCursorPlugin(agentCursor: { }), ); + for (const charPosition of charPositions) { + decs.push( + Decoration.inline(charPosition - 1, charPosition, { + class: "agent-char", + }), + ); + } return DecorationSet.create(doc, decs); }, }, diff --git a/packages/xl-ai/src/prosemirror/agent.ts b/packages/xl-ai/src/prosemirror/agent.ts index 64d1450797..8dad9df24a 100644 --- a/packages/xl-ai/src/prosemirror/agent.ts +++ b/packages/xl-ai/src/prosemirror/agent.ts @@ -268,6 +268,7 @@ export function applyAgentStep(tr: Transaction, step: AgentStep) { selection: { anchor: step.selection.anchor, head: step.selection.head, + inserted: step.type === "insert" || step.type === "replace", }, }); } diff --git a/packages/xl-ai/src/streamTool/asTool.ts b/packages/xl-ai/src/streamTool/asTool.ts index f1f18fc44e..90c86323a3 100644 --- a/packages/xl-ai/src/streamTool/asTool.ts +++ b/packages/xl-ai/src/streamTool/asTool.ts @@ -1,24 +1,30 @@ import { jsonSchema, tool } from "ai"; -import { operationsToStream } from "./callLLMWithStreamTools.js"; +import { + operationsToStream, + validateOperationsRoot, +} from "./callLLMWithStreamTools.js"; import { createStreamToolsArraySchema } from "./jsonSchema.js"; -import { StreamTool } from "./streamTool.js"; +import { StreamTool, StreamToolCall } from "./streamTool.js"; +import { validateOperation } from "./toValidatedOperations.js"; // TODO: remove or implement -export function streamToolAsTool>(streamTool: T) { +export function streamToolAsTool>( + streamTool: T, +) { return tool({ parameters: jsonSchema(streamTool.parameters, { validate: (value) => { - const result = streamTool.validate(value); + const result = streamTool.validate(value as any); if (!result.ok) { return { success: false, error: new Error(result.error) }; } - return { success: true, value: result.value }; + return { success: true, value: result.value as StreamToolCall }; }, }), - execute: async (_value) => { - // console.log("execute", value); - // TODO + execute: async (value) => { + const currentStream = operationsToStream([value]); + return streamTool.execute(currentStream); }, }); } @@ -29,16 +35,27 @@ export function streamToolsAsTool[]>(streamTools: T) { return tool({ parameters: jsonSchema(schema, { validate: (value) => { - const stream = operationsToStream(value); - if (!stream.ok) { - return { success: false, error: new Error(stream.error) }; + const operations = validateOperationsRoot(value); + if (!operations.ok) { + return { success: false, error: new Error(operations.error) }; } - return { success: true, value: stream.value }; + + for (const operation of operations.value) { + const result = validateOperation(operation, streamTools); + if (!result.ok) { + return { success: false, error: new Error(result.error) }; + } + } + + return { success: true, value: operations.value }; }, }), - execute: async (_value) => { - // TODO - // console.log("execute", value); + execute: async (value) => { + let currentStream = operationsToStream(value); + for (const tool of streamTools) { + currentStream = tool.execute(currentStream); + } + return currentStream; }, }); } diff --git a/packages/xl-ai/src/streamTool/callLLMWithStreamTools.ts b/packages/xl-ai/src/streamTool/callLLMWithStreamTools.ts index 78f7dd0089..367ee7af3c 100644 --- a/packages/xl-ai/src/streamTool/callLLMWithStreamTools.ts +++ b/packages/xl-ai/src/streamTool/callLLMWithStreamTools.ts @@ -118,11 +118,11 @@ export async function generateOperations[]>( const ret = await generateObject<{ operations: any }>(options); // because the rest of the codebase always expects a stream, we convert the object to a stream here - const stream = operationsToStream(ret.object); - - if (!stream.ok) { - throw new Error(stream.error); + const operations = validateOperationsRoot(ret.object); + if (!operations.ok) { + throw new Error(operations.error); } + const stream = operationsToStream(operations.value); let _operationsSource: OperationsResult["operationsSource"]; @@ -132,7 +132,7 @@ export async function generateOperations[]>( get operationsSource() { if (!_operationsSource) { _operationsSource = createAsyncIterableStreamFromAsyncIterable( - preprocessOperationsNonStreaming(stream.value, streamTools), + preprocessOperationsNonStreaming(stream, streamTools), ); } return _operationsSource; @@ -143,15 +143,7 @@ export async function generateOperations[]>( }; } -export function operationsToStream[]>( - object: unknown, -): Result< - AsyncIterable<{ - partialOperation: StreamToolCall; - isUpdateToPreviousOperation: boolean; - isPossiblyPartial: boolean; - }> -> { +export function validateOperationsRoot(object: unknown): Result[]> { if ( !object || typeof object !== "object" || @@ -164,20 +156,31 @@ export function operationsToStream[]>( }; } const operations = object.operations; + return { + ok: true, + value: operations, + }; +} + +export function operationsToStream[]>( + operations: StreamToolCall[], +): + AsyncIterable<{ + operation: StreamToolCall; + isUpdateToPreviousOperation: boolean; + isPossiblyPartial: boolean; + }> { async function* singleChunkGenerator() { for (const op of operations) { yield { - partialOperation: op, + operation: op, isUpdateToPreviousOperation: false, isPossiblyPartial: false, }; } } - return { - ok: true, - value: singleChunkGenerator(), - }; + return singleChunkGenerator(), } /** diff --git a/packages/xl-ai/src/streamTool/preprocess.ts b/packages/xl-ai/src/streamTool/preprocess.ts index 0a98ecc216..7bf8f9102f 100644 --- a/packages/xl-ai/src/streamTool/preprocess.ts +++ b/packages/xl-ai/src/streamTool/preprocess.ts @@ -15,7 +15,7 @@ export async function* preprocessOperationsStreaming< T extends StreamTool[], >( operationsStream: AsyncIterable<{ - partialOperation: any; + operation: any; isUpdateToPreviousOperation: boolean; isPossiblyPartial: boolean; }>, @@ -48,7 +48,7 @@ export async function* preprocessOperationsNonStreaming< T extends StreamTool[], >( operationsStream: AsyncIterable<{ - partialOperation: any; + operation: any; isUpdateToPreviousOperation: boolean; isPossiblyPartial: boolean; }>, diff --git a/packages/xl-ai/src/streamTool/toValidatedOperations.ts b/packages/xl-ai/src/streamTool/toValidatedOperations.ts index 0b19668cda..68d1ce0f8e 100644 --- a/packages/xl-ai/src/streamTool/toValidatedOperations.ts +++ b/packages/xl-ai/src/streamTool/toValidatedOperations.ts @@ -1,5 +1,31 @@ import { Result, StreamTool, StreamToolCall } from "./streamTool.js"; +export function validateOperation[]>( + partialOperation: unknown, + streamTools: T, +): Result> { + if ( + !partialOperation || + typeof partialOperation !== "object" || + !("type" in partialOperation) + ) { + return { + ok: false, + error: "Partial operation is not an object", + }; + } + const func = streamTools.find((f) => f.name === partialOperation.type); + + if (!func) { + return { + ok: false, + error: `No matching function for ${partialOperation.type}`, + }; + } + + return func.validate(partialOperation); +} + /** * Transforms the partialObjectStream into a stream of operations (tool calls), or indicates that the operation is invalid. * @@ -9,7 +35,7 @@ import { Result, StreamTool, StreamToolCall } from "./streamTool.js"; */ export async function* toValidatedOperations[]>( partialObjectStream: AsyncIterable<{ - partialOperation: any; + operation: any; isUpdateToPreviousOperation: boolean; isPossiblyPartial: boolean; }>, @@ -20,28 +46,10 @@ export async function* toValidatedOperations[]>( isPossiblyPartial: boolean; }> { for await (const chunk of partialObjectStream) { - const func = streamTools.find( - (f) => f.name === chunk.partialOperation.type, - )!; - - if (!func) { - // Skip operations with no matching function - // console.error("no matching function", chunk.partialOperation); - yield { - operation: { - ok: false, - error: `No matching function for ${chunk.partialOperation.type}`, - }, - isUpdateToPreviousOperation: chunk.isUpdateToPreviousOperation, - isPossiblyPartial: chunk.isPossiblyPartial, - }; - continue; - } - - const operation = func.validate(chunk.partialOperation); + const result = validateOperation(chunk.operation, streamTools); yield { - operation, + operation: result, isUpdateToPreviousOperation: chunk.isUpdateToPreviousOperation, isPossiblyPartial: chunk.isPossiblyPartial, }; diff --git a/packages/xl-ai/src/style.css b/packages/xl-ai/src/style.css index 4b7558d518..35d957c486 100644 --- a/packages/xl-ai/src/style.css +++ b/packages/xl-ai/src/style.css @@ -19,7 +19,7 @@ div[data-type="modification"] { ins, [data-type="modification"] { - background: rgba(24, 122, 220, 0.1); + /* background: rgba(24, 122, 220, 0.1); */ border-bottom: 2px solid rgba(24, 122, 220, 0.1); color: rgb(20, 95, 170); text-decoration: none; @@ -31,3 +31,38 @@ del, text-decoration: line-through; text-decoration-thickness: 1px; } + +div[data-type="insertion"], +ins { + /* // fade in */ + animation: fadeIn 0.6s ease-in-out; +} + +del { + animation: fadeOut 0.6s ease-in-out; + animation-fill-mode: forwards; +} + +.agent-char { + animation: fadeIn 0.6s linear; + animation-fill-mode: forwards; +} + +@keyframes fadeIn { + from { + opacity: 0; + } + to { + opacity: 1; + } +} + +@keyframes fadeOut { + from { + opacity: 1; + } + to { + opacity: 0; + display: none; + } +}