add document intelligence

docs/6-chat-over-file.md

@@ -2,8 +2,14 @@
 
 Users can utilise this functionality to upload their PDF files through the portal and engage in chat discussions related to the content of those files.
 
-### Setup Azure Cognitive Search index
+Chat with your data utilises the following two Azure Services:
 
+1. [Azure Document Intelligence](https://learn.microsoft.com/en-GB/azure/ai-services/document-intelligence/) for extracting information from documents.
+1. [Azure Cognitive Search](https://learn.microsoft.com/en-GB/azure/search/) for indexing and retrieving information.
+
+### Setup Azure Cognitive Search index and Document Intelligence
+
+1. Create Azure Cognitive Search using the following [link](https://learn.microsoft.com/en-us/azure/search/search-get-started-portal)
 1. Create an index on Azure Cognitive Search with the following schema. You can use the Azure portal to create the following [indexes](https://learn.microsoft.com/en-us/azure/search/vector-search-how-to-create-index?tabs=portal-add-field%2Cpush)
 
 ```
@@ -62,8 +68,29 @@ Users can utilise this functionality to upload their PDF files through the porta
 ```
 
 2. After the index has been created, proceed to modify the env.local file with the appropriate Azure Cognitive Search environment variables.
-3. At this point, you have the capability to generate a new chat session with opting for the `file chat` type. Click on the upload button to to start uploading a PDF file.
-4. Upon the successful completion of the file upload, you are now able to commence the conversation using the provided text box.
+
+```
+# Azure cognitive search is used for chat over your data
+AZURE_SEARCH_API_KEY=
+AZURE_SEARCH_NAME=
+AZURE_SEARCH_INDEX_NAME=
+AZURE_SEARCH_API_VERSION="2023-07-01-Preview"
+```
+
+3. Create Azure Document intelligence using the following [link](https://learn.microsoft.com/en-us/azure/ai-services/document-intelligence/create-document-intelligence-resource?view=doc-intel-3.1.0)
+
+4. After the Document intelligence has been created, proceed to modify the env.local file with the appropriate Document intelligence environment variables.
+
+Note that the file is only preserved for each chat thread
+
+```
+# Azure AI Document Intelligence to extract content from your data
+AZURE_DOCUMENT_INTELLIGENCE_ENDPOINT="https://REGION.api.cognitive.microsoft.com/"
+AZURE_DOCUMENT_INTELLIGENCE_KEY=
+```
+
+4. At this point, you have the capability to generate a new chat session with opting for the `file chat` type. Click on the upload button to to start uploading a PDF file.
+5. Upon the successful completion of the file upload, you are now able to commence the conversation using the provided text box.
 
 ### Things to consider:
 

docs/7-environment-variables.md

@@ -22,3 +22,5 @@ Below are the required environment variables, to be added to the Azure Portal or
 | AZURE_SEARCH_NAME                                                                                           | https://AZURE_SEARCH_NAME.search.windows.net | The deployment name of your Azure Cognitive Search                                                                                     |
 | AZURE_SEARCH_INDEX_NAME                                                                                     |                                              | The index name with [vector search](https://learn.microsoft.com/en-us/azure/search/vector-search-overview) enabled                     |
 | AZURE_SEARCH_API_VERSION                                                                                    | 2023-07-01-Preview                           | API version which supports vector search 2023-07-01-Preview                                                                            |
+| AZURE_DOCUMENT_INTELLIGENCE_ENDPOINT                                                                        | https://REGION.api.cognitive.microsoft.com/  | Endpoint url of the Azure document intelligence. The REGION is specific to your Azure resource location                                |
+| AZURE_DOCUMENT_INTELLIGENCE_KEY                                                                             |                                              | API keys of your Azure Document intelligence resource                                                                                  |

src/.env.example

@@ -32,3 +32,7 @@ AZURE_SEARCH_API_KEY=
 AZURE_SEARCH_NAME=
 AZURE_SEARCH_INDEX_NAME=
 AZURE_SEARCH_API_VERSION="2023-07-01-Preview"
+
+# Azure AI Document Intelligence to extract content from your data
+AZURE_DOCUMENT_INTELLIGENCE_ENDPOINT="https://REGION.api.cognitive.microsoft.com/"
+AZURE_DOCUMENT_INTELLIGENCE_KEY=

src/features/chat/chat-menu/menu-items.tsx

@@ -2,7 +2,7 @@
 import { MenuItem } from "@/components/menu";
 import { Button } from "@/components/ui/button";
 import { SoftDeleteChatThreadByID } from "@/features/chat/chat-services/chat-thread-service";
-import { MessageCircle, Trash } from "lucide-react";
+import { FileText, MessageCircle, Trash } from "lucide-react";
 import { useParams, useRouter } from "next/navigation";
 import { FC } from "react";
 import { ChatThreadModel } from "../chat-services/models";
@@ -30,10 +30,18 @@ export const MenuItems: FC<Prop> = (props) => {
           key={thread.id}
           className="justify-between group/item"
         >
-          <MessageCircle
-            size={16}
-            className={id === thread.id ? " text-brand" : ""}
-          />
+          {thread.chatType === "data" ? (
+            <FileText
+              size={16}
+              className={id === thread.id ? " text-brand" : ""}
+            />
+          ) : (
+            <MessageCircle
+              size={16}
+              className={id === thread.id ? " text-brand" : ""}
+            />
+          )}
+
           <span className="flex gap-2 items-center overflow-hidden flex-1">
             <span className="overflow-ellipsis truncate"> {thread.name}</span>
           </span>

src/features/chat/chat-services/chat-document-service.ts

@@ -1,13 +1,23 @@
 "use server";
 
 import { userHashedId } from "@/features/auth/helpers";
+import { initDBContainer } from "@/features/common/cosmos";
 import { AzureCogSearch } from "@/features/langchain/vector-stores/azure-cog-search/azure-cog-vector-store";
+import {
+  AzureKeyCredential,
+  DocumentAnalysisClient,
+} from "@azure/ai-form-recognizer";
 import { Document } from "langchain/document";
-import { PDFLoader } from "langchain/document_loaders/fs/pdf";
 import { OpenAIEmbeddings } from "langchain/embeddings/openai";
 import { RecursiveCharacterTextSplitter } from "langchain/text_splitter";
 import { nanoid } from "nanoid";
-import { FaqDocumentIndex } from "./models";
+import {
+  CHAT_DOCUMENT_ATTRIBUTE,
+  ChatDocumentModel,
+  FaqDocumentIndex,
+} from "./models";
+
+const MAX_DOCUMENT_SIZE = 20000000;
 
 export const UploadDocument = async (formData: FormData) => {
   const { docs, file, chatThreadId } = await LoadFile(formData);
@@ -20,9 +30,39 @@ export const UploadDocument = async (formData: FormData) => {
 const LoadFile = async (formData: FormData) => {
   const file: File | null = formData.get("file") as unknown as File;
   const chatThreadId: string = formData.get("id") as unknown as string;
-  if (file && file.type === "application/pdf" && file.size < 20000000) {
-    const loader = new PDFLoader(file, { splitPages: false });
-    const docs = await loader.load();
+
+  if (
+    file &&
+    file.type === "application/pdf" &&
+    file.size < MAX_DOCUMENT_SIZE
+  ) {
+    const client = initDocumentIntelligence();
+
+    const blob = new Blob([file], { type: file.type });
+
+    const poller = await client.beginAnalyzeDocument(
+      "prebuilt-document",
+      await blob.arrayBuffer()
+    );
+
+    const { paragraphs } = await poller.pollUntilDone();
+
+    const docs: Document[] = [];
+
+    if (paragraphs) {
+      for (const paragraph of paragraphs) {
+        const doc: Document = {
+          pageContent: paragraph.content,
+          metadata: {
+            file: file.name,
+          },
+        };
+        docs.push(doc);
+      }
+    } else {
+      throw new Error("No content found in document.");
+    }
+
     return { docs, file, chatThreadId };
   }
   throw new Error("Invalid file format or size. Only PDF files are supported.");
@@ -61,6 +101,7 @@ const IndexDocuments = async (
   }
 
   await vectorStore.addDocuments(documentsToIndex);
+  await UpsertChatDocument(file.name, chatThreadId);
 };
 
 export const initAzureSearchVectorStore = () => {
@@ -75,3 +116,33 @@ export const initAzureSearchVectorStore = () => {
 
   return azureSearch;
 };
+
+export const initDocumentIntelligence = () => {
+  const client = new DocumentAnalysisClient(
+    process.env.AZURE_DOCUMENT_INTELLIGENCE_ENDPOINT,
+    new AzureKeyCredential(process.env.AZURE_DOCUMENT_INTELLIGENCE_KEY),
+    {
+      apiVersion: "2022-08-31",
+    }
+  );
+
+  return client;
+};
+
+export const UpsertChatDocument = async (
+  fileName: string,
+  chatThreadID: string
+) => {
+  const modelToSave: ChatDocumentModel = {
+    chatThreadId: chatThreadID,
+    id: nanoid(),
+    userId: await userHashedId(),
+    createdAt: new Date(),
+    type: CHAT_DOCUMENT_ATTRIBUTE,
+    isDeleted: false,
+    name: fileName,
+  };
+
+  const container = await initDBContainer();
+  await container.items.upsert(modelToSave);
+};

src/features/chat/chat-services/models.ts

@@ -1,6 +1,7 @@
 import { AzureCogDocument } from "@/features/langchain/vector-stores/azure-cog-search/azure-cog-vector-store";
 import { Message } from "ai";
 
+export const CHAT_DOCUMENT_ATTRIBUTE = "CHAT_DOCUMENT";
 export const CHAT_THREAD_ATTRIBUTE = "CHAT_THREAD";
 export const MESSAGE_ATTRIBUTE = "CHAT_MESSAGE";
 
@@ -54,3 +55,13 @@ export interface FaqDocumentIndex extends AzureCogDocument {
   pageContent: string;
   metadata: any;
 }
+
+export interface ChatDocumentModel {
+  id: string;
+  name: string;
+  chatThreadId: string;
+  userId: string;
+  isDeleted: boolean;
+  createdAt: Date;
+  type: "CHAT_DOCUMENT";
+}