Advanced-RAG/Rag-Source-Handling.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "4335fb06",
   "metadata": {},
   "source": [
    "<center> <h1>Advanced RAG Concepts : Source Handling - Part 1</h1>"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5732e3be",
   "metadata": {
    "jp-MarkdownHeadingCollapsed": true,
    "tags": []
   },
   "source": [
    "### 1. Packages & Models"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2391d862",
   "metadata": {
    "jp-MarkdownHeadingCollapsed": true,
    "tags": []
   },
   "source": [
    "#### 1.1 Packages"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "4fc2ebad",
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    },
    "scrolled": true,
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: unstructured-client in c:\\users\\administrator\\anaconda3\\envs\\myenv\\lib\\site-packages (0.28.1)\n",
      "Requirement already satisfied: langchain-openai in c:\\users\\administrator\\anaconda3\\envs\\myenv\\lib\\site-packages (0.2.14)\n",
      "Requirement already satisfied: langchain-aws in c:\\users\\administrator\\anaconda3\\envs\\myenv\\lib\\site-packages (0.2.10)\n",
      "Requirement already satisfied: langchain-qdrant in c:\\users\\administrator\\anaconda3\\envs\\myenv\\lib\\site-packages (0.2.0)\n",
      "Requirement already satisfied: aiofiles>=24.1.0 in c:\\users\\administrator\\anaconda3\\envs\\myenv\\lib\\site-packages (from unstructured-client) (24.1.0)\n",
      "Requirement already satisfied: cryptography>=3.1 in c:\\users\\administrator\\anaconda3\\envs\\myenv\\lib\\site-packages (from unstructured-client) (44.0.0)\n",
      "Requirement already satisfied: eval-type-backport<0.3.0,>=0.2.0 in c:\\users\\administrator\\anaconda3\\envs\\myenv\\lib\\site-packages (from unstructured-client) (0.2.2)\n",
      "Requirement already satisfied: httpx>=0.27.0 in c:\\users\\administrator\\appdata\\roaming\\python\\python312\\site-packages (from unstructured-client) (0.27.0)\n",
      "Requirement already satisfied: jsonpath-python<2.0.0,>=1.0.6 in c:\\users\\administrator\\anaconda3\\envs\\myenv\\lib\\site-packages (from unstructured-client) (1.0.6)\n",
      "Requirement already satisfied: nest-asyncio>=1.6.0 in c:\\users\\administrator\\appdata\\roaming\\python\\python312\\site-packages (from unstructured-client) (1.6.0)\n",
      "Requirement already satisfied: pydantic<2.10.0,>=2.9.2 in c:\\users\\administrator\\anaconda3\\envs\\myenv\\lib\\site-packages (from unstructured-client) (2.9.2)\n",
      "Requirement already satisfied: pypdf>=4.0 in c:\\users\\administrator\\anaconda3\\envs\\myenv\\lib\\site-packages (from unstructured-client) (4.3.1)\n",
      "Requirement already satisfied: python-dateutil<3.0.0,>=2.8.2 in c:\\users\\administrator\\appdata\\roaming\\python\\python312\\site-packages (from unstructured-client) (2.9.0.post0)\n",
      "Requirement already satisfied: requests-toolbelt>=1.0.0 in c:\\users\\administrator\\anaconda3\\envs\\myenv\\lib\\site-packages (from unstructured-client) (1.0.0)\n",
      "Requirement already satisfied: typing-inspect<0.10.0,>=0.9.0 in c:\\users\\administrator\\anaconda3\\envs\\myenv\\lib\\site-packages (from unstructured-client) (0.9.0)\n",
      "Requirement already satisfied: langchain-core<0.4.0,>=0.3.27 in c:\\users\\administrator\\anaconda3\\envs\\myenv\\lib\\site-packages (from langchain-openai) (0.3.28)\n",
      "Requirement already satisfied: openai<2.0.0,>=1.58.1 in c:\\users\\administrator\\anaconda3\\envs\\myenv\\lib\\site-packages (from langchain-openai) (1.58.1)\n",
      "Requirement already satisfied: tiktoken<1,>=0.7 in c:\\users\\administrator\\anaconda3\\envs\\myenv\\lib\\site-packages (from langchain-openai) (0.8.0)\n",
      "Requirement already satisfied: boto3>=1.35.74 in c:\\users\\administrator\\anaconda3\\envs\\myenv\\lib\\site-packages (from langchain-aws) (1.35.90)\n",
      "Requirement already satisfied: numpy<3,>=1.26.0 in c:\\users\\administrator\\anaconda3\\envs\\myenv\\lib\\site-packages (from langchain-aws) (2.0.1)\n",
      "Requirement already satisfied: qdrant-client<2.0.0,>=1.10.1 in c:\\users\\administrator\\anaconda3\\envs\\myenv\\lib\\site-packages (from langchain-qdrant) (1.12.2)\n",
      "Requirement already satisfied: botocore<1.36.0,>=1.35.90 in c:\\users\\administrator\\anaconda3\\envs\\myenv\\lib\\site-packages (from boto3>=1.35.74->langchain-aws) (1.35.90)\n",
      "Requirement already satisfied: jmespath<2.0.0,>=0.7.1 in c:\\users\\administrator\\anaconda3\\envs\\myenv\\lib\\site-packages (from boto3>=1.35.74->langchain-aws) (1.0.1)\n",
      "Requirement already satisfied: s3transfer<0.11.0,>=0.10.0 in c:\\users\\administrator\\anaconda3\\envs\\myenv\\lib\\site-packages (from boto3>=1.35.74->langchain-aws) (0.10.4)\n",
      "Requirement already satisfied: cffi>=1.12 in c:\\users\\administrator\\appdata\\roaming\\python\\python312\\site-packages (from cryptography>=3.1->unstructured-client) (1.16.0)\n",
      "Requirement already satisfied: anyio in c:\\users\\administrator\\appdata\\roaming\\python\\python312\\site-packages (from httpx>=0.27.0->unstructured-client) (4.4.0)\n",
      "Requirement already satisfied: certifi in c:\\users\\administrator\\appdata\\roaming\\python\\python312\\site-packages (from httpx>=0.27.0->unstructured-client) (2024.7.4)\n",
      "Requirement already satisfied: httpcore==1.* in c:\\users\\administrator\\appdata\\roaming\\python\\python312\\site-packages (from httpx>=0.27.0->unstructured-client) (1.0.5)\n",
      "Requirement already satisfied: idna in c:\\users\\administrator\\appdata\\roaming\\python\\python312\\site-packages (from httpx>=0.27.0->unstructured-client) (3.7)\n",
      "Requirement already satisfied: sniffio in c:\\users\\administrator\\appdata\\roaming\\python\\python312\\site-packages (from httpx>=0.27.0->unstructured-client) (1.3.1)\n",
      "Requirement already satisfied: h11<0.15,>=0.13 in c:\\users\\administrator\\appdata\\roaming\\python\\python312\\site-packages (from httpcore==1.*->httpx>=0.27.0->unstructured-client) (0.14.0)\n",
      "Requirement already satisfied: PyYAML>=5.3 in c:\\users\\administrator\\appdata\\roaming\\python\\python312\\site-packages (from langchain-core<0.4.0,>=0.3.27->langchain-openai) (6.0.1)\n",
      "Requirement already satisfied: jsonpatch<2.0,>=1.33 in c:\\users\\administrator\\anaconda3\\envs\\myenv\\lib\\site-packages (from langchain-core<0.4.0,>=0.3.27->langchain-openai) (1.33)\n",
      "Requirement already satisfied: langsmith<0.3,>=0.1.125 in c:\\users\\administrator\\anaconda3\\envs\\myenv\\lib\\site-packages (from langchain-core<0.4.0,>=0.3.27->langchain-openai) (0.2.6)\n",
      "Requirement already satisfied: packaging<25,>=23.2 in c:\\users\\administrator\\appdata\\roaming\\python\\python312\\site-packages (from langchain-core<0.4.0,>=0.3.27->langchain-openai) (24.1)\n",
      "Requirement already satisfied: tenacity!=8.4.0,<10.0.0,>=8.1.0 in c:\\users\\administrator\\anaconda3\\envs\\myenv\\lib\\site-packages (from langchain-core<0.4.0,>=0.3.27->langchain-openai) (9.0.0)\n",
      "Requirement already satisfied: typing-extensions>=4.7 in c:\\users\\administrator\\anaconda3\\envs\\myenv\\lib\\site-packages (from langchain-core<0.4.0,>=0.3.27->langchain-openai) (4.12.2)\n",
      "Requirement already satisfied: distro<2,>=1.7.0 in c:\\users\\administrator\\anaconda3\\envs\\myenv\\lib\\site-packages (from openai<2.0.0,>=1.58.1->langchain-openai) (1.9.0)\n",
      "Requirement already satisfied: jiter<1,>=0.4.0 in c:\\users\\administrator\\anaconda3\\envs\\myenv\\lib\\site-packages (from openai<2.0.0,>=1.58.1->langchain-openai) (0.8.2)\n",
      "Requirement already satisfied: tqdm>4 in c:\\users\\administrator\\anaconda3\\envs\\myenv\\lib\\site-packages (from openai<2.0.0,>=1.58.1->langchain-openai) (4.66.5)\n",
      "Requirement already satisfied: annotated-types>=0.6.0 in c:\\users\\administrator\\anaconda3\\envs\\myenv\\lib\\site-packages (from pydantic<2.10.0,>=2.9.2->unstructured-client) (0.7.0)\n",
      "Requirement already satisfied: pydantic-core==2.23.4 in c:\\users\\administrator\\anaconda3\\envs\\myenv\\lib\\site-packages (from pydantic<2.10.0,>=2.9.2->unstructured-client) (2.23.4)\n",
      "Requirement already satisfied: six>=1.5 in c:\\users\\administrator\\appdata\\roaming\\python\\python312\\site-packages (from python-dateutil<3.0.0,>=2.8.2->unstructured-client) (1.16.0)\n",
      "Requirement already satisfied: grpcio>=1.41.0 in c:\\users\\administrator\\anaconda3\\envs\\myenv\\lib\\site-packages (from qdrant-client<2.0.0,>=1.10.1->langchain-qdrant) (1.68.1)\n",
      "Requirement already satisfied: grpcio-tools>=1.41.0 in c:\\users\\administrator\\anaconda3\\envs\\myenv\\lib\\site-packages (from qdrant-client<2.0.0,>=1.10.1->langchain-qdrant) (1.68.1)\n",
      "Requirement already satisfied: portalocker<3.0.0,>=2.7.0 in c:\\users\\administrator\\anaconda3\\envs\\myenv\\lib\\site-packages (from qdrant-client<2.0.0,>=1.10.1->langchain-qdrant) (2.10.1)\n",
      "Requirement already satisfied: urllib3<3,>=1.26.14 in c:\\users\\administrator\\appdata\\roaming\\python\\python312\\site-packages (from qdrant-client<2.0.0,>=1.10.1->langchain-qdrant) (2.2.2)\n",
      "Requirement already satisfied: requests<3.0.0,>=2.0.1 in c:\\users\\administrator\\appdata\\roaming\\python\\python312\\site-packages (from requests-toolbelt>=1.0.0->unstructured-client) (2.32.3)\n",
      "Requirement already satisfied: regex>=2022.1.18 in c:\\users\\administrator\\anaconda3\\envs\\myenv\\lib\\site-packages (from tiktoken<1,>=0.7->langchain-openai) (2024.11.6)\n",
      "Requirement already satisfied: mypy-extensions>=0.3.0 in c:\\users\\administrator\\anaconda3\\envs\\myenv\\lib\\site-packages (from typing-inspect<0.10.0,>=0.9.0->unstructured-client) (1.0.0)\n",
      "Requirement already satisfied: pycparser in c:\\users\\administrator\\appdata\\roaming\\python\\python312\\site-packages (from cffi>=1.12->cryptography>=3.1->unstructured-client) (2.22)\n",
      "Requirement already satisfied: protobuf<6.0dev,>=5.26.1 in c:\\users\\administrator\\anaconda3\\envs\\myenv\\lib\\site-packages (from grpcio-tools>=1.41.0->qdrant-client<2.0.0,>=1.10.1->langchain-qdrant) (5.29.2)\n",
      "Requirement already satisfied: setuptools in c:\\users\\administrator\\anaconda3\\envs\\myenv\\lib\\site-packages (from grpcio-tools>=1.41.0->qdrant-client<2.0.0,>=1.10.1->langchain-qdrant) (72.1.0)\n",
      "Requirement already satisfied: h2<5,>=3 in c:\\users\\administrator\\anaconda3\\envs\\myenv\\lib\\site-packages (from httpx[http2]>=0.20.0->qdrant-client<2.0.0,>=1.10.1->langchain-qdrant) (4.1.0)\n",
      "Requirement already satisfied: jsonpointer>=1.9 in c:\\users\\administrator\\appdata\\roaming\\python\\python312\\site-packages (from jsonpatch<2.0,>=1.33->langchain-core<0.4.0,>=0.3.27->langchain-openai) (3.0.0)\n",
      "Requirement already satisfied: orjson<4.0.0,>=3.9.14 in c:\\users\\administrator\\anaconda3\\envs\\myenv\\lib\\site-packages (from langsmith<0.3,>=0.1.125->langchain-core<0.4.0,>=0.3.27->langchain-openai) (3.10.12)\n",
      "Requirement already satisfied: pywin32>=226 in c:\\users\\administrator\\appdata\\roaming\\python\\python312\\site-packages (from portalocker<3.0.0,>=2.7.0->qdrant-client<2.0.0,>=1.10.1->langchain-qdrant) (306)\n",
      "Requirement already satisfied: charset-normalizer<4,>=2 in c:\\users\\administrator\\appdata\\roaming\\python\\python312\\site-packages (from requests<3.0.0,>=2.0.1->requests-toolbelt>=1.0.0->unstructured-client) (3.3.2)\n",
      "Requirement already satisfied: colorama in c:\\users\\administrator\\appdata\\roaming\\python\\python312\\site-packages (from tqdm>4->openai<2.0.0,>=1.58.1->langchain-openai) (0.4.6)\n",
      "Requirement already satisfied: hyperframe<7,>=6.0 in c:\\users\\administrator\\anaconda3\\envs\\myenv\\lib\\site-packages (from h2<5,>=3->httpx[http2]>=0.20.0->qdrant-client<2.0.0,>=1.10.1->langchain-qdrant) (6.0.1)\n",
      "Requirement already satisfied: hpack<5,>=4.0 in c:\\users\\administrator\\anaconda3\\envs\\myenv\\lib\\site-packages (from h2<5,>=3->httpx[http2]>=0.20.0->qdrant-client<2.0.0,>=1.10.1->langchain-qdrant) (4.0.0)\n",
      "Note: you may need to restart the kernel to use updated packages.\n"
     ]
    }
   ],
   "source": [
    "%pip install -U unstructured-client langchain-openai langchain-aws langchain-qdrant"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2b8a9386",
   "metadata": {
    "jp-MarkdownHeadingCollapsed": true,
    "tags": []
   },
   "source": [
    "#### 1.2 Models"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "fcd0c71e",
   "metadata": {},
   "source": [
    "##### 1.2.1 Embedding Model on LM Studio"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "917fab92",
   "metadata": {},
   "outputs": [],
   "source": [
    "from langchain_openai import OpenAIEmbeddings\n",
    "\n",
    "embedding = OpenAIEmbeddings(\n",
    "    openai_api_base=\"http://localhost:1234/v1\", \n",
    "    api_key=\"type-anything-here\",\n",
    "    model=\"text-embedding-bge-large-en-v1.5\",\n",
    "    check_embedding_ctx_length=False\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1a8361cb",
   "metadata": {},
   "source": [
    "##### 1.2.2 LLM on Amazon Bedrock"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "df193b44",
   "metadata": {},
   "outputs": [],
   "source": [
    "import yaml\n",
    "\n",
    "with open('../secrets.yml', 'r') as file:\n",
    "    credentials = yaml.safe_load(file)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "e1b6ec5a",
   "metadata": {},
   "outputs": [],
   "source": [
    "from langchain_aws import ChatBedrock\n",
    "\n",
    "llm = ChatBedrock(\n",
    "    model_id=\"us.meta.llama3-3-70b-instruct-v1:0\",\n",
    "    region=\"us-east-1\",\n",
    "    aws_access_key_id=credentials[\"bedrock\"][\"access_key\"],\n",
    "    aws_secret_access_key=credentials[\"bedrock\"][\"secret_key\"]\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f356da7e",
   "metadata": {},
   "source": [
    "#### 1.3 Disable Unnecessary Logging"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "82635289",
   "metadata": {},
   "outputs": [],
   "source": [
    "import logging\n",
    "logger = logging.getLogger()\n",
    "logger.setLevel(logging.CRITICAL)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "53241c1c",
   "metadata": {
    "tags": []
   },
   "source": [
    "### 2. Data Preparation"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ac831abb",
   "metadata": {
    "tags": []
   },
   "source": [
    "#### 2.1 Sample Documents"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "aecf9474",
   "metadata": {},
   "source": [
    "For this tutorial, our sample documents are wikipedia articles (downloaded as pdf) for the following companies :\n",
    "* [BNP Paribas](https://en.wikipedia.org/wiki/BNP_Paribas)\n",
    "* [STMicroelectronics](https://en.wikipedia.org/wiki/STMicroelectronics)\n",
    "* [Palantir Technologies](https://en.wikipedia.org/wiki/Palantir_Technologies)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0927ab6d",
   "metadata": {},
   "source": [
    "#### 2.2 OCR & Chunking"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ac029755",
   "metadata": {},
   "source": [
    "##### 2.2.1 Set up Unstructured Client"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "6e305a1f",
   "metadata": {},
   "outputs": [],
   "source": [
    "import unstructured_client\n",
    "from unstructured_client.models import operations, shared\n",
    "\n",
    "client = unstructured_client.UnstructuredClient(\n",
    "    api_key_auth=credentials[\"unstructured\"][\"api_key\"],\n",
    "    server_url=\"https://api.unstructured.io/general/v0/general\",\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "62a9ecb8",
   "metadata": {},
   "source": [
    "##### 2.2.2 Define Processing Function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "507d4c4e",
   "metadata": {},
   "outputs": [],
   "source": [
    "import random\n",
    "\n",
    "def process_document(filepath):\n",
    "    with open(filepath, \"rb\") as f:\n",
    "        files = shared.Files(\n",
    "            content=f.read(),\n",
    "            file_name=filepath\n",
    "        )\n",
    "\n",
    "    req = operations.PartitionRequest(\n",
    "        partition_parameters = shared.PartitionParameters(\n",
    "            files=files,\n",
    "            languages=[\"eng\"],\n",
    "            chunking_strategy=\"by_title\",\n",
    "            multipage_sections=True\n",
    "        )\n",
    "    )\n",
    "\n",
    "    res = client.general.partition(request=req)\n",
    "    element_dicts = [element for element in res.elements]\n",
    "    return element_dicts\n",
    "\n",
    "\n",
    "def show_random_chunks(elements):\n",
    "    view = random.sample(range(len(elements)), 3)\n",
    "    for i in view:\n",
    "        e = elements[i]\n",
    "        print(f\"Chunk N°{i+1} : {e['text']}\\n\\n\"+20*\"=\"+\"\\n\\n\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "da273ce5",
   "metadata": {},
   "source": [
    "##### 2.2.3 Process Sample Documents"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "601d04c4",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Chunk N°23 : Financial data\n",
      "\n",
      "In 2022, total revenues of €50.4 billion represent an increase of 9% compared to 2021, BNP Paribas remains at the top of the French banks' ranking in terms of activity. During this year, BNP Paribas Group net income attributable to equity holders increased to 7.5% (to 10.2 billion euros). The geographic breakdown of Net Banking Income (NBI) at the end of\n",
      "\n",
      "====================\n",
      "\n",
      "\n",
      "Chunk N°47 : The Maison dorée, home to Parisian operations of BNP Paribas CIB, with the registered office of BNP Paribas in the background\n",
      "\n",
      "====================\n",
      "\n",
      "\n",
      "Chunk N°56 : dollar-dominated transactions. The fine exceeded the bank's $6.4 billion 2013 annual income and the $1.1 billion it previously had allocated for the anticipated fine.[60][61]\n",
      "\n",
      "====================\n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "bnp_elements = process_document(\"BNP_Paribas.pdf\")\n",
    "show_random_chunks(bnp_elements)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "id": "7b3b8e47",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'type': 'CompositeElement',\n",
       " 'element_id': '8fcec7946ce4f03517e36154f4fdb916',\n",
       " 'text': '% WIKIPEDIA & The Free Encyclopedia\\n\\nBNP Paribas',\n",
       " 'metadata': {'filetype': 'application/pdf',\n",
       "  'languages': ['eng'],\n",
       "  'page_number': 1,\n",
       "  'orig_elements': 'eJy1UU1LxDAQ/SshoKdtaNI23XpTXKGIUlDwUJaSNtM20C/aLG5Z/O8mrQuinkSP7828mXnz0hOGBlrodKYkvkI4d33JPBY6YSSZ44sInKhkriNFKdiWU8oZ4A3CLWghhRZGc8JF349SdULDtOBGzP1BZzWoqtaGYZ7PjOaDflVS14alPOCGHXrVaatLU0o94m8QZZRE+w06Y+aGxLXYD1yy/Y7XfkPgaZ40tNZHoo7QPA2iAPxmChI0FFr1XVY0YpqyYexz02bkkeuZFbhUDeh5gEWbPODl3K46iGrxlGLoKmxXDIbJukObw2g92OEajtYlvkAv8X2c7G7ja3SJnmtAdyMA2nXFXDT9AFIJO/e8Jm7NKHvd1wxcmXNZbEsHBKeOL0PPEbKQjs8BqKQl8CD/xwxCYihGOaFrBisOOQnXnweE/oCX/t9lwH2XsT/K4OYxQYkYVS6mz79+VroxV+zfAQkc2+A=',\n",
       "  'filename': 'BNP_Paribas.pdf'}}"
      ]
     },
     "execution_count": 61,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "bnp_elements[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "021c1594",
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Chunk N°22 : ST Ericsson was a multinational manufacturer of wireless products and semiconductors, supplying to mobile device manufacturers.[8] ST-Ericsson was a 50/50 joint venture of STMicroelectronics and Ericsson established on February 3, 2009, and dissolved on August 2, 2013. Headquartered in Geneva, Switzerland, it was a fabless company, outsourcing semiconductor manufacturing to foundry companies.\n",
      "\n",
      "====================\n",
      "\n",
      "\n",
      "Chunk N°37 : \"Numonyx\" in 2008. A new manufacturing facility for silicon carbide (SiC) substrates of 150 mm should open here in 2023.[23]\n",
      "\n",
      "====================\n",
      "\n",
      "\n",
      "Chunk N°17 : On 8 December 1994, the company completed its initial public offering on the Paris and New York stock exchanges. Owner Thomson SA sold its stake in the company in 1998 when the company also listed on the Italian Bourse in Milan. In 2002, Motorola and TSMC joined ST and Philips in a new technology partnership. The Crolles 2 Alliance was created with a new 12\" wafer manufacturing facility located in Crolles, France. In 2005, chief executive officer Pasquale Pistorio was succeeded by Carlo Bozotti,\n",
      "\n",
      "====================\n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "stm_elements = process_document(\"STMicroelectronics.pdf\")\n",
    "show_random_chunks(stm_elements)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "fbee03c4",
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Chunk N°7 : Palantir Foundry is used for data integration and analysis by corporate clients such as Morgan Stanley, Merck KGaA, Airbus, Wejo, Lilium, PG&E and Fiat Chrysler Automobiles.[11] Palantir Apollo is a platform to facilitate continuous integration/continuous delivery (CI/CD) across all environments.[12][13]\n",
      "\n",
      "====================\n",
      "\n",
      "\n",
      "Chunk N°113 : On September 28, 2020, Amnesty International released a report criticizing Palantir failure to conduct human rights due diligence around its contracts with ICE. Concerns around Palantir's rights record were being scrutinized for contributing to human rights violations of asylum-seekers and migrants.[144][145]\n",
      "\n",
      "\"HHS Protect Now\" and privacy concerns (since 2020)\n",
      "\n",
      "====================\n",
      "\n",
      "\n",
      "Chunk N°120 : Board of directors\n",
      "\n",
      "As of December 2024, the board of directors of Palantir includes:[151]\n",
      "\n",
      "=\n",
      "\n",
      "Alex Karp, CEO of Palantir\n",
      "\n",
      "[]\n",
      "\n",
      "Alexander Moore, co-founder and former CEO of NodePrime\n",
      "\n",
      "[]\n",
      "\n",
      "Alexandra Schiff, former reporter of The Wall Street Journal\n",
      "\n",
      "[]\n",
      "\n",
      "Stephen Cohen, co-founder and president of Palantir\n",
      "\n",
      "Peter Thiel, co-founder of PayPal, Palantir and Founders Fund\n",
      "\n",
      "=\n",
      "\n",
      "=\n",
      "\n",
      "Lauren Friedman Stat, former Fractional Chief Administration Officer at\n",
      "\n",
      "====================\n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "palantir_elements = process_document(\"Palantir_Technologies.pdf\")\n",
    "show_random_chunks(palantir_elements)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "379ee18c",
   "metadata": {
    "tags": []
   },
   "source": [
    "#### 2.3 Vectorization"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f8212a89-33f1-488d-b2a4-83f5cb7ab589",
   "metadata": {},
   "source": [
    "##### 2.3.1 Set up Qdrant client"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "8bb76a76-e0a9-4946-bf9c-66fbf5b56416",
   "metadata": {},
   "outputs": [],
   "source": [
    "from qdrant_client import QdrantClient\n",
    "from qdrant_client.http.models import Distance, VectorParams\n",
    "from langchain_qdrant import QdrantVectorStore, RetrievalMode\n",
    "from langchain_core.documents import Document\n",
    "\n",
    "qdrant_clt = QdrantClient(\":memory:\")\n",
    "\n",
    "qdrant_clt.create_collection(\n",
    "    collection_name=\"embedding_tutorial\",\n",
    "    vectors_config=VectorParams(size=1024, distance=Distance.COSINE),\n",
    ")\n",
    "\n",
    "qdrant = QdrantVectorStore(\n",
    "    embedding=embedding,\n",
    "    client=qdrant_clt,\n",
    "    collection_name=\"embedding_tutorial\",\n",
    "    retrieval_mode=RetrievalMode.DENSE,\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5a6b6805-da58-4971-ae97-2f8a4f5565f0",
   "metadata": {},
   "source": [
    "##### 2.3.2 Define Vectorization Function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "468deec3-01a1-4da5-8f2f-295e19890267",
   "metadata": {},
   "outputs": [],
   "source": [
    "def vectorize_elements(elements):\n",
    "    documents = []\n",
    "    ids = []\n",
    "    for e in elements:\n",
    "        document = Document(page_content=e[\"text\"], metadata=e[\"metadata\"])\n",
    "        documents.append(document)\n",
    "        ids.append(e[\"element_id\"])\n",
    "    qdrant.add_documents(documents=documents, ids=ids)\n",
    "    print(f\"Successfully upserted {len(documents)} vectors to Qdrant !\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e10bff48",
   "metadata": {},
   "source": [
    "##### 2.3.3 Vectorize Sample Documents"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "id": "66d307a5-c9ef-46ab-b830-0040c863828a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Successfully upserted 66 vectors to Qdrant !\n"
     ]
    }
   ],
   "source": [
    "vectorize_elements(bnp_elements)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "id": "80de03ef-eeee-4d7f-bac8-b04fbc306c33",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Successfully upserted 60 vectors to Qdrant !\n"
     ]
    }
   ],
   "source": [
    "vectorize_elements(stm_elements)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "id": "d6a2a362",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Successfully upserted 127 vectors to Qdrant !\n"
     ]
    }
   ],
   "source": [
    "vectorize_elements(palantir_elements)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "fb18a206",
   "metadata": {
    "tags": []
   },
   "source": [
    "### 3. RAG Pipeline"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1646820b",
   "metadata": {
    "tags": []
   },
   "source": [
    "#### 3.1 Retrieve"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "id": "6401f2c5-7dca-42f9-a48a-481f8e9e37df",
   "metadata": {},
   "outputs": [],
   "source": [
    "from langchain_core.runnables import RunnablePassthrough\n",
    "\n",
    "retriever = qdrant.as_retriever(search_kwargs={\"k\": 10})\n",
    "\n",
    "retriever_runnable = {\"query\": RunnablePassthrough(), \"docs\": retriever}"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "95e33815",
   "metadata": {
    "tags": []
   },
   "source": [
    "#### 3.2 Augment"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "id": "6cc0ccf0-b64a-4d45-88bc-60cab9455149",
   "metadata": {},
   "outputs": [],
   "source": [
    "from langchain_core.runnables import RunnableLambda\n",
    "\n",
    "def augment_context(input_dict):\n",
    "    query = input_dict[\"query\"]\n",
    "    docs = input_dict[\"docs\"]\n",
    "    context = \"\"\n",
    "    sourcing = {}\n",
    "    for i, doc in enumerate(docs):\n",
    "        sourcing[i+1] = {\"text\": doc.page_content, **doc.metadata}\n",
    "        context += f\"Snippet {i+1}: {doc.page_content}\" + \"<end of snippet>\\n\"\n",
    "    return {\"query\": query, \"context\": context, \"sourcing\": sourcing}\n",
    "\n",
    "augmenter_runnable = RunnableLambda(augment_context)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "89718f47",
   "metadata": {
    "tags": []
   },
   "source": [
    "#### 3.3 Generate"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "id": "37b64d3b-5a07-45e0-8b82-d3d8d2623bd5",
   "metadata": {},
   "outputs": [],
   "source": [
    "from langchain_core.prompts import ChatPromptTemplate\n",
    "from langchain_core.output_parsers import StrOutputParser\n",
    "from textwrap import dedent\n",
    "\n",
    "system_prompt = \"\"\"\n",
    "You are a data-based question answering assistant. \n",
    "You only base your answers on the data provided to you along with the question.\n",
    "You always refer to the specific origin of the information you used for each part of your answer.\n",
    "You are given a question and a set of snippets of texts to help you answer the question.\n",
    "The snippets are provided to you in the following format: 'Snippet <number>: <text>'. A '<end of snippet>' tag signals the end of a snippet.\n",
    "If you use information from a snippet to make a statement, your statement (the answer to the question) should be followed by the snippet number in this format : **(<number>)**.\n",
    "For example, if you use 'Snippet 4' to answer a question, your answer should look like this : '<answer> **(4)**'.\n",
    "If you combined multiple snippets (e.g. 5 and 3) to answer a question, your answer should look like this : '<first statement> **(5)**, <second statement> **(3)**, ...'.\n",
    "If you don't have enough information to answer, just say the following failure message : 'There wasn't enough information to answer the question.'.\n",
    "\"\"\"\n",
    "\n",
    "user_prompt = \"\"\"\n",
    "Question: {query}\n",
    "Context: {context}\n",
    "Answer: \n",
    "\"\"\"\n",
    "\n",
    "prompt = ChatPromptTemplate([\n",
    "    (\"system\", dedent(system_prompt)),\n",
    "    (\"user\", dedent(user_prompt)),\n",
    "])\n",
    "\n",
    "generator_runnable = prompt | llm | StrOutputParser()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "id": "8cf968f6-5710-40d8-8dfb-8699952e470a",
   "metadata": {},
   "outputs": [],
   "source": [
    "chain = retriever_runnable | augmenter_runnable | generator_runnable"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "id": "e501e762-b614-4598-a3c9-2c0f32013533",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "BNP Paribas was founded in 2000 from the merger of two of France's foremost financial institutions, Banque Nationale de Paris (BNP) and Paribas **(2)**, with the merger taking place on 23 May 2000 **(1)**, **(3)**, **(5)**.\n"
     ]
    }
   ],
   "source": [
    "result = chain.invoke(\"When was BNP Paribas founded ?\")\n",
    "print(result)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "544692a1-ca8c-4d7f-be16-28f54eed58c5",
   "metadata": {
    "tags": []
   },
   "source": [
    "#### 3.4 Source"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "67d38540-d4a6-4ac6-9dda-eb7c2d14379f",
   "metadata": {
    "tags": []
   },
   "source": [
    "##### 3.4.1 Prettify Sourced Answer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "id": "5eef6571-dd0b-4977-bc92-95fd957e187f",
   "metadata": {},
   "outputs": [],
   "source": [
    "import re\n",
    "\n",
    "superscripts = \"⁰¹²³⁴⁵⁶⁷⁸⁹\"\n",
    "\n",
    "def int_to_superscript(integer):\n",
    "    return ''.join(superscripts[int(d)] for d in str(integer))\n",
    "\n",
    "def superscript_to_int(superscript):\n",
    "    return ''.join(str([c for c in superscripts].index(d)) for d in superscript)\n",
    "\n",
    "def prettify_sources(text):\n",
    "    pattern = r\"\\*\\*\\((\\d+)\\)\\*\\*\"\n",
    "    matches = re.findall(pattern, text)\n",
    "    integers = list(map(int, matches))\n",
    "    superscript_map = {}\n",
    "    seen = set()\n",
    "    i = 0\n",
    "    for integer in integers:\n",
    "        if integer not in seen:\n",
    "            superscript_map[integer] = int_to_superscript(i + 1)\n",
    "            i += 1\n",
    "            seen.add(integer)\n",
    "    replaced_text = text\n",
    "    for i, old_value in enumerate(matches):\n",
    "        new_value = superscript_map[int(old_value)]\n",
    "        replaced_text = replaced_text.replace(f\" **({old_value})**\", new_value, 1)\n",
    "    \n",
    "    superscript_to_old = {v: k for k, v in superscript_map.items()}\n",
    "    \n",
    "    return replaced_text, superscript_to_old"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "aca6ff73-0b2a-4d40-8efe-74c26f853127",
   "metadata": {},
   "source": [
    "##### 3.4.2 Sourced RAG Chain"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "id": "abe1cad2-3ed8-424f-b322-865f3586abf2",
   "metadata": {},
   "outputs": [],
   "source": [
    "def show_source(input_dict):\n",
    "    answer = input_dict[\"answer\"]\n",
    "    sourcing = input_dict[\"sourcing\"]\n",
    "    new_answer, source_matching = prettify_sources(answer)\n",
    "    new_answer += \"\\n\\nSources:\\n\"\n",
    "    for superscript, integer in source_matching.items():\n",
    "        src_meta = sourcing[integer]\n",
    "        new_answer += superscript + src_meta[\"filename\"] + f\", page n°{src_meta['page_number']}.\\n\"\n",
    "    return new_answer.strip()\n",
    "\n",
    "source_runnable = RunnableLambda(show_source)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "id": "971c1f67-8f40-43b7-b9d0-f20b395c54dd",
   "metadata": {},
   "outputs": [],
   "source": [
    "srag_chain = retriever_runnable | augmenter_runnable | RunnablePassthrough.assign(answer=generator_runnable) | source_runnable"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1aaaf92c",
   "metadata": {},
   "source": [
    "##### 3.4.3 Tests/Examples"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "id": "80cb7e5b-8865-4eb7-a9b9-31812635d009",
   "metadata": {},
   "outputs": [],
   "source": [
    "result = srag_chain.invoke(\"When and how was STMicroelectronics founded ?\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "id": "fb49f7f6-8a1f-4b9c-a386-5538d64bb610",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "STMicroelectronics was founded in 1987¹ from the merger of two state-owned semiconductor corporations: Thomson Semiconducteurs of France and SGS Microelettronica of Italy¹, which were two government-owned semiconductor companies². At the time of the merger, the new corporation was named SGS-THOMSON³ and was led by chief executive officer Pasquale Pistorio³. The company took its current name of STMicroelectronics in May 1998³.\n",
      "\n",
      "Sources:\n",
      "¹STMicroelectronics.pdf, page n°1.\n",
      "²STMicroelectronics.pdf, page n°1.\n",
      "³STMicroelectronics.pdf, page n°2.\n"
     ]
    }
   ],
   "source": [
    "print(result)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "id": "b1bc4fd4",
   "metadata": {},
   "outputs": [],
   "source": [
    "result = srag_chain.invoke(\"What are the major products of Palantir Technologies ?\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "id": "9ae92903",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The major products of Palantir Technologies are Palantir Gotham, Palantir Foundry, Palantir Apollo, and Palantir AIP¹. Additionally, the company also offers Palantir Metropolis², which is used by hedge funds, banks, and financial services firms³. Palantir Foundry is used for data integration and analysis⁴, Palantir Gotham is an intelligence and defense tool¹, and Palantir Apollo is a platform to facilitate continuous integration/continuous delivery (CI/CD)⁴.\n",
      "\n",
      "Sources:\n",
      "¹Palantir_Technologies.pdf, page n°2.\n",
      "²Palantir_Technologies.pdf, page n°7.\n",
      "³Palantir_Technologies.pdf, page n°7.\n",
      "⁴Palantir_Technologies.pdf, page n°2.\n"
     ]
    }
   ],
   "source": [
    "print(result)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5a6b481e",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}