From 4a7de8641cf5bf080c262353fd194ff182934414 Mon Sep 17 00:00:00 2001 From: Shohei Maeda <11495867+smaeda-ks@users.noreply.github.com> Date: Sun, 22 Jan 2023 07:43:02 +0900 Subject: [PATCH 1/2] Move parsing logic to server-side --- package-lock.json | 16 +++++++++++++++- package.json | 1 + pages/api/generate.ts | 44 ++++++++++++++++++++++++++++++++++++++++--- pages/index.tsx | 44 ++----------------------------------------- 4 files changed, 59 insertions(+), 46 deletions(-) diff --git a/package-lock.json b/package-lock.json index 49f2d05f..0b1daadd 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,5 +1,5 @@ { - "name": "twitter-bio-generator", + "name": "twitterbio", "lockfileVersion": 2, "requires": true, "packages": { @@ -10,6 +10,7 @@ "@heroicons/react": "^2.0.13", "@tailwindcss/forms": "^0.5.3", "@vercel/analytics": "^0.1.8", + "eventsource-parser": "^0.0.5", "framer-motion": "^8.4.3", "next": "latest", "react": "18.2.0", @@ -732,6 +733,14 @@ "node": ">=6" } }, + "node_modules/eventsource-parser": { + "version": "0.0.5", + "resolved": "https://registry.npmjs.org/eventsource-parser/-/eventsource-parser-0.0.5.tgz", + "integrity": "sha512-BAq82bC3ZW9fPYYZlofXBOAfbpmDzXIOsj+GOehQwgTUYsQZ6HtHs6zuRtge7Ph8OhS6lNH1kJF8q9dj17RcmA==", + "engines": { + "node": ">=12" + } + }, "node_modules/fast-glob": { "version": "3.2.12", "resolved": "https://registry.npmjs.org/fast-glob/-/fast-glob-3.2.12.tgz", @@ -2051,6 +2060,11 @@ "integrity": "sha512-k0er2gUkLf8O0zKJiAhmkTnJlTvINGv7ygDNPbeIsX/TJjGJZHuh9B2UxbsaEkmlEo9MfhrSzmhIlhRlI2GXnw==", "dev": true }, + "eventsource-parser": { + "version": "0.0.5", + "resolved": "https://registry.npmjs.org/eventsource-parser/-/eventsource-parser-0.0.5.tgz", + "integrity": "sha512-BAq82bC3ZW9fPYYZlofXBOAfbpmDzXIOsj+GOehQwgTUYsQZ6HtHs6zuRtge7Ph8OhS6lNH1kJF8q9dj17RcmA==" + }, "fast-glob": { "version": "3.2.12", "resolved": "https://registry.npmjs.org/fast-glob/-/fast-glob-3.2.12.tgz", diff --git a/package.json b/package.json index 8c2eebac..fe3ce025 100644 --- a/package.json +++ b/package.json @@ -11,6 +11,7 @@ "@heroicons/react": "^2.0.13", "@tailwindcss/forms": "^0.5.3", "@vercel/analytics": "^0.1.8", + "eventsource-parser": "^0.0.5", "framer-motion": "^8.4.3", "next": "latest", "react": "18.2.0", diff --git a/pages/api/generate.ts b/pages/api/generate.ts index 7fa89eec..90312ee3 100644 --- a/pages/api/generate.ts +++ b/pages/api/generate.ts @@ -1,4 +1,9 @@ import type { NextRequest } from "next/server"; +import { + createParser, + ParsedEvent, + ReconnectInterval, +} from "eventsource-parser"; if (!process.env.OPENAI_API_KEY) { throw new Error("Missing env var from OpenAI"); @@ -17,6 +22,9 @@ const handler = async (req: NextRequest): Promise => { return new Response("No prompt in the request", { status: 400 }); } + const encoder = new TextEncoder(); + const decoder = new TextDecoder(); + const payload = { model: "text-davinci-003", prompt, @@ -38,11 +46,41 @@ const handler = async (req: NextRequest): Promise => { body: JSON.stringify(payload), }); - const data = res.body; + let counter = 0; + const stream = new ReadableStream({ + async start(controller) { + // callback + function onParse(event: ParsedEvent | ReconnectInterval) { + if (event.type === "event") { + const data = event.data; + // https://beta.openai.com/docs/api-reference/completions/create#completions/create-stream + if (data === "[DONE]") { + controller.close(); + return; + } + const json = JSON.parse(data); + const text = json.choices[0].text; + if (counter < 2 && (text.match(/\n/) || []).length) { + // this is a prefix character (i.e., "\n\n"), do nothing + return; + } + const queue = encoder.encode(text); + controller.enqueue(queue); + counter++; + } + } - return new Response(data, { - headers: { "Content-Type": "application/json; charset=utf-8" }, + // stream response (SSE) from OpenAI may be fragmented into multiple chunks + // this ensures we properly read chunks and invoke an event for each SSE event stream + const parser = createParser(onParse); + // https://web.dev/streams/#asynchronous-iteration + for await (const chunk of res.body as any) { + parser.feed(decoder.decode(chunk)); + } + }, }); + + return new Response(stream); }; export default handler; diff --git a/pages/index.tsx b/pages/index.tsx index 2a22e98a..ae80a166 100644 --- a/pages/index.tsx +++ b/pages/index.tsx @@ -59,51 +59,11 @@ const Home: NextPage = () => { const decoder = new TextDecoder(); let done = false; - let tempState = ""; - while (!done) { const { value, done: doneReading } = await reader.read(); done = doneReading; - const newValue = decoder - .decode(value) - .replaceAll("data: ", "") - .split("\n\n") - .filter(Boolean); - - if (tempState) { - newValue[0] = tempState + newValue[0]; - tempState = ""; - } - - newValue.forEach((newVal) => { - if (newVal === "[DONE]") { - return; - } - - try { - const json = JSON.parse(newVal) as { - id: string; - object: string; - created: number; - choices?: { - text: string; - index: number; - logprobs: null; - finish_reason: null | string; - }[]; - model: string; - }; - - if (!json.choices?.length) { - throw new Error("Something went wrong."); - } - - const choice = json.choices[0]; - setGeneratedBios((prev) => prev + choice.text); - } catch (error) { - tempState = newVal; - } - }); + const chunkValue = decoder.decode(value); + setGeneratedBios((prev) => prev + chunkValue); } setLoading(false); From b7ed528ab025d3bd06349a9aa67d26581d096922 Mon Sep 17 00:00:00 2001 From: Shohei Maeda <11495867+smaeda-ks@users.noreply.github.com> Date: Mon, 23 Jan 2023 03:29:28 +0900 Subject: [PATCH 2/2] handle errors --- pages/api/generate.ts | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/pages/api/generate.ts b/pages/api/generate.ts index 90312ee3..e6ece065 100644 --- a/pages/api/generate.ts +++ b/pages/api/generate.ts @@ -58,15 +58,20 @@ const handler = async (req: NextRequest): Promise => { controller.close(); return; } - const json = JSON.parse(data); - const text = json.choices[0].text; - if (counter < 2 && (text.match(/\n/) || []).length) { - // this is a prefix character (i.e., "\n\n"), do nothing - return; + try { + const json = JSON.parse(data); + const text = json.choices[0].text; + if (counter < 2 && (text.match(/\n/) || []).length) { + // this is a prefix character (i.e., "\n\n"), do nothing + return; + } + const queue = encoder.encode(text); + controller.enqueue(queue); + counter++; + } catch (e) { + // maybe parse error + controller.error(e); } - const queue = encoder.encode(text); - controller.enqueue(queue); - counter++; } }