Levothyrox_sentiment_analysis.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Levothyrox sentiment analysis",
      "provenance": [],
      "toc_visible": true,
      "authorship_tag": "ABX9TyNEr93o49E9/W4YKU2SvfVV",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/yohan-th/Levothyscan/blob/master/Levothyrox_sentiment_analysis.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "E3VKlLmdvIs_",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import re\n",
        "import sys\n",
        "import csv\n",
        "import threading\n",
        "import timeit\n",
        "import time\n",
        "import pandas as pd\n",
        "import urllib.request\n",
        "\n",
        "\n",
        "# Parametre\n",
        "search = \"levothyrox\"\n",
        "rubrique = \"18*sante\"\n",
        "debug = False\n",
        "save_tmp_out = True"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "zv5wScOKvj8Z",
        "colab_type": "text"
      },
      "source": [
        "# **Tools**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "6ITPKLniviN5",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "class GetAllPages_topic_Thread(threading.Thread):\n",
        "\n",
        "    def __init__(self, turl, page, nb_page, sujet):\n",
        "        threading.Thread.__init__(self)\n",
        "        self.url = turl\n",
        "        self.page = page\n",
        "        self.nb_page = nb_page\n",
        "        self.sujet = sujet\n",
        "\n",
        "    def run(self):\n",
        "        global all_msg\n",
        "        if debug:\n",
        "          print(\"[Ouverture URL de \\\"\" + self.sujet + \"\\\" page \" + str(page) + \"] : \" + str(self.url))\n",
        "        \n",
        "        html = get_html(url, 5)\n",
        "        if html == -1:\n",
        "          print(f\"[Error get messages] --> abort page {url}\")\n",
        "          return\n",
        "       \n",
        "        only_messages = re.search('<div id=\"topic\" >(.*?)<div class=\"bottom_action_topic_menu\">', html, re.MULTILINE | re.DOTALL).group(1)\n",
        "        messages_page = re.findall('class=\"md-topic_post(.*?)/table>', only_messages, re.MULTILINE | re.DOTALL)\n",
        "\n",
        "        for message in messages_page:\n",
        "          if re.match('.*data-id_user.*', message, re.DOTALL):\n",
        "              user = re.search('data-id_user.*?>(.+?)<', message).group(1)\n",
        "          elif re.match('.*itemprop=\"name\"', message, re.DOTALL):\n",
        "              try:\n",
        "                user = re.search('itemprop=\"name\".+?>(.+?)<', message).group(1)\n",
        "              except:\n",
        "                print(html)\n",
        "          elif re.match('.*Profil supprimé.*', message, re.DOTALL):\n",
        "              user = \"Profil supprimé\"\n",
        "          else:\n",
        "              user = \"[ERROR_Encodage_user_unknown]\"\n",
        "          date = re.search('Posté le ([0-9/]+)', message).group(1)\n",
        "          if re.match('.*itemprop=\"citation\".*', message, re.DOTALL):\n",
        "            message = re.sub('itemprop=\"citation\".+?</span></span>', '', message, flags=re.DOTALL)\n",
        "          text = re.search('itemprop=\"text\" hidden>(.*?)</span><div>', message, re.MULTILINE | re.DOTALL).group(1)\n",
        "          text = clean_message(text)\n",
        "          all_msg = all_msg.append({'date':date, 'user':user, 'text':text, 'url':self.url}, ignore_index=True)\n",
        "\n",
        "        if debug:\n",
        "          print(\"[\" + str(len(messages_page)) + \" new msg sur \\\"\" + self.sujet + \"\\\" de la page \" + str(self.page) + \" sur \" + self.nb_page + \"]\")\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "icvDfrtzwTZQ",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import unicodedata\n",
        "\n",
        "def clean_message(msg):\n",
        "    msg = re.sub('&#039;', '\\'', msg)#apostrophe\n",
        "    msg = re.sub(',', ' ', msg)  # Pour un decoupage correct sur excel\n",
        "    msg = re.sub('[>\\r\\n]+', ' ', msg) #Saut de ligne\n",
        "    msg = re.sub(':\\w+:', ' ', msg) #les smiley :happy:\n",
        "    msg = re.sub('http\\://.+?\\.html', '', msg) #les liens copi\n",
        "    msg = re.sub('<img.*?/>', ' ', msg)  #suppr les images\n",
        "    msg = re.sub('<br.*?>', ' ', msg) #suppr les balise br\n",
        "    msg = re.sub('<a (.*?)</a>', ' ', msg) #suppr les liens externe\n",
        "    msg = re.sub('</?span.*?>', ' ', msg)\n",
        "    msg = re.sub('</?table.*?>', ' ', msg)\n",
        "    msg = re.sub('</?[a-z][a-z]?>', ' ', msg) #</i> <lu> et bien d'autre\n",
        "    msg = re.sub('&[a-z#0-9]{1,4};', ' ', msg) #&#034; &nbsp; &euro; &gt; &lt;\n",
        "    msg = re.sub('\\[#[0-9]+ size=[0-9]+\\]', ' ', msg)\n",
        "    msg = re.sub('</?strong>', ' ', msg)\n",
        "    msg = re.sub('</?div>?', ' ', msg)\n",
        "\n",
        "    #Decommenter les lignes suivantes pour clean plus exhaustif\n",
        "    #while re.search(\" ['\\w^.><?!)(/@*_&%:+\\-]{0,4} \", msg):\n",
        "    #    msg = re.sub(\" ['\\w^.><?!)(/@*_&%:;+\\-]{0,4} \", ' ', msg)\n",
        "    #msg = unicodedata.normalize('NFD', msg).encode('ascii', 'ignore')  # suppr les accents\n",
        "    return(msg)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "v5OdOG8-wW7w",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def get_nbr_page(html):\n",
        "    list_pages = re.search('pagination_main_visible(.+?)/div', html).group(1)\n",
        "    if re.match(r\".*href.*\", list_pages):\n",
        "        return(re.findall(\"\\\">([0-9]+)<\", html)[-1])\n",
        "    else:\n",
        "        return(\"1\")"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "wSCFeXPzwYf4",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def get_html(url:str, max_attempt:int):\n",
        "  attempt = 1\n",
        "\n",
        "  while (attempt <= max_attempt):\n",
        "    try:\n",
        "      with urllib.request.urlopen(url) as response:\n",
        "        html = response.read().decode('utf-8')\n",
        "        return html\n",
        "    except OSError as e:\n",
        "      print(f\"[Error {e.code}] {e.reason} : {url}\")\n",
        "      if e.code == 503:\n",
        "        time.sleep(60)\n",
        "      time.sleep(1)\n",
        "      attempt += 1\n",
        "  return -1"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "NaH-GaINwa3g",
        "colab_type": "text"
      },
      "source": [
        "# Script"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ucp9ew_qwlgA",
        "colab_type": "text"
      },
      "source": [
        "Récupéation des adresses url de l'ensemble des topics"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "TJWEldcRwi34",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "print(\"Recherche de <\"+search+\"> dans la rubrique <\"+rubrique+\">\")\n",
        "if debug:\n",
        "    print('http://forum.doctissimo.fr/search_result.php?post_cat_list='+rubrique+'&search='+search+'&resSearch=250')\n",
        "with urllib.request.urlopen('http://forum.doctissimo.fr/search_result.php?post_cat_list='+rubrique+'&search='+search+'&resSearch=250') as response:\n",
        "  html = response.read().decode('utf-8')\n",
        "if re.match(r\".*aucune réponse n'a été trouvée.*\", html, re.MULTILINE|re.DOTALL):\n",
        "    print(\"La recherche de <\"+search+\"> dans la rubrique <\"+rubrique+\"> donne aucun résultat\")\n",
        "    sys.exit()\n",
        "nb_page_topic = get_nbr_page(html)\n",
        "\n",
        "print(nb_page_topic+\" page(s) de 250 topics sur le sujet <\"+search+\"> dans la rubrique <\"+rubrique+\">\")\n",
        "\n",
        "all_topics_url = []\n",
        "page = 1\n",
        "if debug:\n",
        "    nb_page_topic = \"1\"\n",
        "while page <= int(nb_page_topic):\n",
        "    print(\"telechargement de page \" + str(page) + \" sur \" + nb_page_topic)\n",
        "    if debug:\n",
        "        print('http://forum.doctissimo.fr/search_result.php?post_cat_list='+rubrique+'&search='+search+'&resSearch=250&page='+str(page))\n",
        "    with urllib.request.urlopen('http://forum.doctissimo.fr/search_result.php?post_cat_list='+rubrique+'&search='+search+'&resSearch=250&page='+str(page)) as response:\n",
        "        html = response.read().decode('utf-8')\n",
        "    topics = re.findall(r\"</?t.*?sujet ligne_booleen(.+?)</tr>\", html, re.MULTILINE | re.DOTALL)\n",
        "    for topic in topics:\n",
        "        if debug:\n",
        "            print(re.search(r\"href=\\\"(.+?)\\\"\", topic).group(1))\n",
        "        all_topics_url.append(re.search(r\"href=\\\"(.+?)\\\"\", topic).group(1))\n",
        "    page += 1\n",
        "print(\"nb total de topic = \" + str(len(all_topics_url)))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "oOhy1UWx3GTC",
        "colab_type": "text"
      },
      "source": [
        "Récupération des adresses url des pages de chaque topic et téléchargement de chaque page dans un thread séparé"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "h7mp_aH92-l0",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "start = timeit.default_timer()\n",
        "\n",
        "all_msg = pd.DataFrame(columns=['date','user', 'text', 'url'])\n",
        "threadList = []\n",
        "nb_topic = 0\n",
        "\n",
        "for url in all_topics_url:\n",
        "    if debug:\n",
        "        time.sleep(2)\n",
        "\n",
        "    html = get_html(url, 5)\n",
        "    if html == -1:\n",
        "      print(f\"[Error get nb page] --> abort topic {url}\")\n",
        "      continue \n",
        "\n",
        "    sujet_topic = re.search(\"forum.doctissimo.fr/sante/.+/(.*?)sujet_\", url).group(1)\n",
        "    nb_page_topic = get_nbr_page(html)\n",
        "    if debug:\n",
        "        print(\"topic \\\"\"+sujet_topic+\"\\\" avec \"+str(nb_page_topic)+\" page(s)\")\n",
        "    \n",
        "    page = 1\n",
        "    while page <= int(nb_page_topic):\n",
        "        clean_url = re.search(r\"(.*)_\", url).group(1)\n",
        "        newthread = GetAllPages_topic_Thread(clean_url + \"_\" + str(page) + \".htm\", page, nb_page_topic, sujet_topic)\n",
        "        newthread.start()\n",
        "        time.sleep(0.1)\n",
        "        threadList.append(newthread)\n",
        "        page += 1\n",
        "\n",
        "    if (nb_topic % 100 == 0):\n",
        "      print(str(nb_topic) + \" topics extrait sur \" + str(len(all_topics_url)) + \". Messages récoltés : \" + str(len(all_msg)))\n",
        "    \n",
        "    if len(all_msg) > 1000 and save_tmp_out == True:\n",
        "        all_msg.to_csv(\"tmp_out.csv\", sep=',', encoding='utf-8', index=False)\n",
        "        print(\"Fichier temporaire save --> tmp_out.csv\")\n",
        "        save_tmp_out = False\n",
        "\n",
        "    nb_topic += 1\n",
        "\n",
        "print(\"Attente des threads\")\n",
        "for curThread in threadList :\n",
        "    curThread.join()\n",
        "\n",
        "all_msg.to_csv(\"out.csv\", sep=',', encoding='utf-8', index=False)\n",
        "\n",
        "stop = timeit.default_timer()\n",
        "m, s = divmod(stop - start, 60)\n",
        "h, m = divmod(m, 60)\n",
        "print(str(len(all_msg)) + \" messages total récoltés en %dh %02dmin et %02ds\" % (h, m, s))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "rOW0_3Hf3Uhf",
        "colab_type": "text"
      },
      "source": [
        "# Test"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "VA2zZ2fm3a0e",
        "colab_type": "text"
      },
      "source": [
        "Pour vérifier que les regex sont ok pour extraire :\n",
        "*   user\n",
        "*   date\n",
        "*   text\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "kDqo8gsO3YuA",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from google.colab import files\n",
        "import re\n",
        "import pandas as pd\n",
        "\n",
        "url = \"http://forum.doctissimo.fr/sante/arthrose-os/maigrir-sujet_149370_1.htm\" # profil supprimé + citation normol\n",
        "url = \"http://forum.doctissimo.fr/sante/thyroide-problemes-endocrinologiques/endocrinologue-belgique-sujet_160644_1.htm\" # encodage user avec space authorisé (Susanne in F)\n",
        "url = \"http://forum.doctissimo.fr/sante/thyroide-problemes-endocrinologiques/supportez-thyroxine-sanofi-sujet_171008_1.htm\" #avec hidden dans code devant user name\n",
        "url = \"http://forum.doctissimo.fr/sante/thyroide-problemes-endocrinologiques/demande-renseignements-sujet_171692_1.htm\" #encodage citaiton different\n",
        "\n",
        "url = \"http://forum.doctissimo.fr/sante/regles-problemes-gynecologiques/retart-regles-sujet_222033_1.htm\" # encodage citation différent\n",
        "\n",
        "url = \"http://forum.doctissimo.fr/sante/thyroide-problemes-endocrinologiques/probleme-couple-tyroide-sujet_152716_2.htm\" #message de fay41ft le 24/04/2006 \"...\" cité absent sur la page web mais présent dans le tableau ??\n",
        "\n",
        "df = pd.DataFrame(columns=['date','user', 'text', 'url'])\n",
        "\n",
        "try:\n",
        "    with urllib.request.urlopen(url) as rep:\n",
        "        html = rep.read().decode('utf-8')\n",
        "except (http.client.IncompleteRead) as e:\n",
        "    html = e.partial.decode('utf-8')\n",
        "#print(html)\n",
        "\n",
        "#with open(\"1_all_website.html\", \"w\") as file:\n",
        "#    file.write(html) \n",
        "#files.download('1_all_website.html')\n",
        "\n",
        "\n",
        "only_messages = re.search('<div id=\"topic\" >(.*?)<div class=\"bottom_action_topic_menu\">', html, re.MULTILINE | re.DOTALL).group(1)\n",
        "#with open(\"2_only_messages.html\", \"w\") as file:\n",
        "#     file.write(only_messages) \n",
        "#files.download('2_only_messages.html')\n",
        "\n",
        "messages_page = re.findall('class=\"md-topic_post(.*?)/table>', only_messages, re.MULTILINE | re.DOTALL)\n",
        "\n",
        "for message in messages_page:\n",
        "    if re.match('.*data-id_user.*', message, re.DOTALL):\n",
        "        user = re.search('data-id_user.+?>(.+?)<', message).group(1)\n",
        "    elif re.match('.*itemprop=\"name\"', message, re.DOTALL):\n",
        "        user = re.search('itemprop=\"name\".*?>(.+?)<', message).group(1) #parfois hidden est rajouté dans le code source donc .+? après name\n",
        "    elif re.match('.*Profil supprimé.*', message, re.DOTALL):\n",
        "        user = \"Profil supprimé\"\n",
        "    else:\n",
        "        user = \"[ERROR_Encodage_user_unknown]\"\n",
        "    date = re.search('Posté le ([0-9/]+)', message).group(1)\n",
        "    if re.match('.*itemprop=\"citation\".*', message, re.DOTALL):\n",
        "        message = re.sub('itemprop\\=\\\"citation\\\".+?</span><span', '', message, flags=re.DOTALL)\n",
        "    text = re.search('itemprop=\"text\" hidden>(.*?)</span>[<div>|<span itemprop=\"author\"]', message, re.MULTILINE | re.DOTALL).group(1)\n",
        "    text = clean_message(text)\n",
        "    df = df.append({'date':date, 'user':user, 'text':text, 'url':url}, ignore_index=True)\n",
        "\n",
        "print(str(len(messages_page)))\n",
        "df\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "fjevIg676TQn",
        "colab_type": "text"
      },
      "source": [
        "# Data Cleaning"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ziq9zEAj6XlN",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import pandas as pd\n",
        "import numpy as np\n",
        "import matplotlib.pyplot as plt\n",
        "import seaborn \n",
        "%matplotlib inline"
      ],
      "execution_count": 0,
      "outputs": []
    }
  ]
}