02 - Scraping Instagram - stats from all the pictures.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from bs4 import BeautifulSoup\n",
    "from bs4 import SoupStrainer\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import urllib\n",
    "import urllib.parse\n",
    "import re\n",
    "import os\n",
    "import subprocess\n",
    "import requests\n",
    "import random\n",
    "import time\n",
    "from requests import get\n",
    "\n",
    "account =  ['resortalyeska',\n",
    "            'skiheavenly',\n",
    "            'kirkwoodmtn',\n",
    "            'mammothmountain',\n",
    "            'northstar_california',\n",
    "            'squawalpine',\n",
    "            'sugarbowlresort',\n",
    "            'arapahoe_basin',\n",
    "            'aspensnowmass',\n",
    "            'aspenco',\n",
    "            'beavercreek',\n",
    "            'breckenridgemtn',\n",
    "            'coppermtn',\n",
    "            'skicrestedbutte',\n",
    "            'keystone_resort',\n",
    "            'lovelandskiarea',\n",
    "            'skipowderhorn',\n",
    "            'silvertonmtn',\n",
    "            'steamboatresort',\n",
    "            'tellurideski',\n",
    "            'vailmtn',\n",
    "            'winterparkresort',\n",
    "            'schweitzer_mountain',\n",
    "            'sunvalley',\n",
    "            'tamarackresort',\n",
    "            'bigskyresort',\n",
    "            'bridgerbowl',\n",
    "            'mtroseskitahoe',\n",
    "            'skitaos',\n",
    "            'mthoodmeadows',\n",
    "            'altaskiarea',\n",
    "            'brightonresort',\n",
    "            'deervalleyresort',\n",
    "            'pcski',\n",
    "            'powdermountain',\n",
    "            'snowbasinresort',\n",
    "            'snowbird',\n",
    "            'solitudemtn',\n",
    "            'jaypeakresort',\n",
    "            'killingtonmtn',\n",
    "            'madriverglen',\n",
    "            'stowemt',\n",
    "            'crystalmountain',\n",
    "            'mtbakerskiarea',\n",
    "            'stevenspass',           \n",
    "            'grandtargheeresort',\n",
    "            'jacksonhole',\n",
    "            'sundanceresort',\n",
    "            'bogusbasin',\n",
    "            'summitatsnoqualmie',\n",
    "            'skipurg',\n",
    "            'brianheadresort',\n",
    "            'sierra_at_tahoe',\n",
    "            'skieaglecrest',\n",
    "            'silvermountain',\n",
    "            'skisantafe',\n",
    "            'skithebeav',\n",
    "            'brundagemtn',\n",
    "            'skieaglepoint',\n",
    "            'whitepass',\n",
    "            'losttrailskiarea',\n",
    "            'smugglersnotchvt',\n",
    "            'redlodge',\n",
    "            'monarchmountain',\n",
    "            'sugarbush_vt',\n",
    "            'skiwhitefish',\n",
    "            'wolfcreekski',\n",
    "            'mtbachelor',\n",
    "            'whistlerblackcomb',\n",
    "            'revelstoke',\n",
    "            'ferniealpineresort',\n",
    "            'whitewaterskiresort',\n",
    "            'redresort',\n",
    "            'sunshinevillage',\n",
    "            'kickinghorsemtn',\n",
    "            'powderkingmountainresort',\n",
    "            'skilouise',\n",
    "            'panoramaresort',\n",
    "            'castlemountainresort',\n",
    "            'skibigwhite',\n",
    "            'cypressmtn',\n",
    "            'silverstarmtnresort',\n",
    "            'sunpeaksresort',\n",
    "            'mountwashington',\n",
    "            'timberlinelodge',\n",
    "            'skihomewood',\n",
    "            'eldoramtnresort',\n",
    "            'bearvalleymountainupdates',\n",
    "            'diamondpeak',\n",
    "            '49degreesnorth',\n",
    "            'lookoutpass',\n",
    "            'discoveryskiarea',\n",
    "            'pebbleskiarea',\n",
    "            'sunlightmountainresort',\n",
    "            'skibowl',\n",
    "            'dodgeridge',\n",
    "            'anthonylakes',\n",
    "            'marmotbasin',\n",
    "            'apexmtnresort']\n",
    "\n",
    "for i in range(0, len(account)):\n",
    "    \n",
    "    # open the VPN-connection and wait for 60 seconds for the program to actually connect\n",
    "    vpn = subprocess.Popen('c:\\\\Program Files\\\\Trust.Zone VPN Client\\\\trustzone_x64.exe')\n",
    "    time.sleep(60)\n",
    "\n",
    "    # collect and print the public IP address to make sure the VPN worked\n",
    "    public_ip = get('https://api.ipify.org').text\n",
    "    print('The public IP address is:',public_ip)\n",
    "    \n",
    "    # set the URL to the Instagram account\n",
    "    url = 'https://www.instagram.com/' + str(account[i])\n",
    "\n",
    "    # make a folder to save the data\n",
    "    path = urllib.parse.urlparse(url).path\n",
    "    folder_name = path.replace('/', '')\n",
    "    folder_path = 'images/{}/'.format(folder_name)\n",
    "\n",
    "    account_data = pd.read_csv(os.path.join(folder_path, str(account[i]) + \"_pictures.csv\"), sep = ',')\n",
    "    number_input = account_data.shape[0]\n",
    "    \n",
    "    image_url_list = account_data.values.tolist()\n",
    "    \n",
    "    only_meta_tags = SoupStrainer('meta')\n",
    "\n",
    "    image_info_list = []\n",
    "    image_info = ()\n",
    "    image_counter = 0\n",
    "\n",
    "    for j in range(len(image_url_list)):\n",
    "        video_link = str()\n",
    "        image_link = str()\n",
    "        likes_image = str()\n",
    "        comments_image = str()\n",
    "        hashtags_image = 0\n",
    "\n",
    "        page = requests.get(image_url_list[j][2])\n",
    "        soup_image = BeautifulSoup(page.text, 'html.parser', parse_only = only_meta_tags)\n",
    "\n",
    "        for link in soup_image.find_all('meta'):\n",
    "            metaprop = link.get('property', '')\n",
    "\n",
    "            if metaprop == \"og:video\":\n",
    "                video_link = link.get('content')\n",
    "\n",
    "            if metaprop == \"og:image\":\n",
    "                image_link = link.get('content')\n",
    "\n",
    "            if metaprop == \"og:description\":\n",
    "                description_info = link.get('content')\n",
    "\n",
    "                # parse the number of likes (get all the text before Likes or vind-ik-leuks)\n",
    "                likes_image_str = re.search('(\\d.+)?(?=Likes|vind-ik-leuks)', str(description_info)).group(0).strip()\n",
    "\n",
    "                # get rid of the k or K (thousands) and commas and convert to integers\n",
    "                likes_image_temp = re.sub('[kK]+', '', likes_image_str)\n",
    "                if likes_image_temp == likes_image_str:\n",
    "                    likes_image = int(float(re.sub('[,|.]+', '', likes_image_temp)))\n",
    "                else:\n",
    "                    likes_image = int(float(re.sub('[,|.]+', '.', likes_image_temp)) * 1000)\n",
    "                \n",
    "                # parse the number of comments (get all the text between the comma and Comments or reacties)\n",
    "                comments_image_str = re.search('(?:s,)(.*?)(?=Comments|reacties)', str(description_info)).group(0).strip()[3:]\n",
    "                \n",
    "                # get rid of the k or K (thousands) and commas and convert to integers\n",
    "                comments_image_temp = re.sub('[kK]+', '', comments_image_str)\n",
    "                if comments_image_temp == comments_image_str:\n",
    "                    comments_image = int(float(re.sub('[,|.]+', '', comments_image_temp)))\n",
    "                else:\n",
    "                    comments_image = int(float(re.sub('[,|.]+', '.', comments_image_temp)) * 1000)\n",
    "\n",
    "            if metaprop == \"og:title\":\n",
    "                title_info = link.get('content')\n",
    "                date_post = re.search('(Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec).(\\d.+)(?=at)', str(title_info)).group(0).strip()\n",
    "                time_post = re.search('(\\d\\sat.*$)', str(title_info)).group(0).strip()[5:]\n",
    "            if metaprop == \"instapp:hashtags\":\n",
    "                hashtags_image += 1\n",
    "                \n",
    "        image_counter += 1\n",
    "        account_id = i + 1\n",
    "        image_id = (i + 1) * 100000 + image_counter\n",
    "        followers_account = image_url_list[j][3]\n",
    "        date_scrape = time.strftime(\"%b %#d, %Y\")\n",
    "        image_info = (account_id, image_id, followers_account, title_info, date_post, time_post, date_scrape, description_info, likes_image, comments_image, hashtags_image, video_link, image_link)\n",
    "        image_info_list.append(image_info)\n",
    "    \n",
    "        if image_counter%100 == 0:\n",
    "            print('Stats of', image_counter, 'images processed')\n",
    "    \n",
    "        # wait after every picture between 0 and 2 seconds to minimize the chance of being banned\n",
    "        # wait after every 100 pictures for period between 10 and 20 seconds to minimize the chance of being banned\n",
    "        if image_counter%100 == 0:\n",
    "            period_100 = np.random.uniform(10, 20)\n",
    "            time.sleep(period_100)\n",
    "        else:\n",
    "            period_1 = np.random.uniform(0, 2)\n",
    "            time.sleep(period_1)\n",
    "            \n",
    "    likes_comments = pd.DataFrame([[x[0], x[1], x[2], x[3], x[4], x[5], x[6], x[7], x[8], x[9], x[10], x[11], x[12]] for x in image_info_list])\n",
    "    likes_comments.columns = ['account_id', 'image_id', 'followers_account', 'title_info', 'date_post', 'time_post', 'date_scrape', 'description', 'likes_image', 'comments_image', 'hashtags_image', 'video_link', 'image_link']\n",
    "    likes_comments.to_csv(os.path.join(folder_path, str(account[i]) + \"_stats.csv\"), sep = ',', index = False)\n",
    "    \n",
    "    number_output = likes_comments.shape[0]\n",
    "    \n",
    "    print('Number of records of input and output are the same:', number_input == number_output)\n",
    "    \n",
    "    vpn.terminate()\n",
    "    time.sleep(60)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}