input.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "from sklearn.decomposition import LatentDirichletAllocation\n",
    "\n",
    "import gensim\n",
    "from gensim import corpora, models, similarities\n",
    "\n",
    "import nltk\n",
    "from nltk.stem.porter import PorterStemmer\n",
    "from nltk.stem.lancaster import LancasterStemmer\n",
    "from nltk.corpus import stopwords\n",
    "\n",
    "from pymongo import MongoClient\n",
    "from time import time\n",
    "from collections import Counter, defaultdict\n",
    "from textblob import TextBlob\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import operator\n",
    "import re\n",
    "import os\n",
    "import pickle"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Load Resume"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "with open('BryantBiggs_mod.txt', 'r') as f:\n",
    "    new_resume = f.read().strip()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'BRYANT J. BIGGS\\n                          3134 Avalon Way           *            Bloomingdale, New Jersey            *            bryantbiggs@gmail.com\\n\\nEDUCATION\\nPurdue University - West Lafayette, IN\\nBachelor of Science - M.E.T.\\nReceived - December 19, 2009\\t\\t\\t\\nProgram GPA - 3.28/4.00\\n\\nArizona State University - Tempe, AZ\\nBachelor of Science - Software Eng.\\nExpected - February, 2017\\t\\t\\nProgram GPA - 4.00/4.00\\n\\t    \\n\\nEXPERIENCE\\nDATA SCIENTIST\\n         Metis - New York City, NY. June, 2016 - Current\\nAn immersive 12 week data science boot camp to learn and apply concepts such as data science, data mining, supervised (SVM, decision trees, random forests, KNN) and unsupervised (NLP, KMeans) machine learning, statistical inference, Bayesian statistics, regression analysis, data visualization, and cloud computing.\\nTechnologies used/taught:\\n  ? Python - Scikit-learn, SciPy, NumPy, Pandas, BeautifulSoup, Selenium, Matplotlib, Flask, Django\\n  ? Javascript - D3.js, jQuery \\n  ? SQL and NoSQL databases - PostgreSQL, MongoDB\\n \\t   ? Apache - Hadoop, MapReduce, and Spark.\\nAPPLICATION ENGINEER\\n         Saint-Gobain Performance Plastics - Wayne, NJ. June, 2014 - Current\\n* Product design for PTFE based plain bearings used primarily in automotive and high-performance bicycle applications.\\n* New product application development from initial prototype through launch within a global engineering team. \\n* Development of enterprise applications to assist in rapid engineering drawing/document creation, automated script to create design based recommendation document for customers, material usage calculator, Inventor based script to automate detail drawings, and macro based 3D parametric models to create internal and external tool sets.\\n* Maintained multiple Access database used in project tracking. Designed and created new database used by North American HR team to review employee performance metrics and career growth. \\nPRODUCT ENGINEER\\n         Wyman-Gordon - Grafton, MA. December, 2012 - May, 2014\\n* Three-dimensional design of forging process models, forging dies, heat treatment and machining fixtures.\\n* Finite elemental analysis (FEA) of material flow, cavity fill, defect analysis, tool stress, and dead-zone mitigation.\\n* Continuous cost reduction efforts with emphasis on material utilization and reduction in operational steps.\\n* Process and product documentation improvement - work instructions and master router.\\n* Led new product introduction to educate facility on new products.\\n DESIGN ENGINEER\\n         Stalcop U.S. L.P. - Thorntown, IN. March, 2010 - November, 2012\\n* Tool design research and development for cold-forming ferrous and non-ferrous metals. \\n* Process design for high and low volume manufacturing as well as prototype development. \\n* Finite elemental analysis (FEA) of material flow and tool stress using Deform 2D/3D/F3.\\n* Quality documentation including PPAP documentation, PFMEA, capability studies, gage R&R, process control plan, process/inspection flow diagram, and sample inspection reports. \\nSOFTWARE PROFICIENCIES\\n\\nMicrosoft Office\\nSolidWorks\\nAutodesk Inventor \\nVisual Studio & VBA\\nPython\\nSQL\\nMachine Learning\\nMongoDB \\nPostgreSQL \\nNumPy, SciPy\\nGithub'"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "new_resume"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# City, State Abbreviation List"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "with open(r'pkl/cities.pkl', 'rb') as infile:\n",
    "       cities = pickle.load(infile)\n",
    "\n",
    "with open(r'pkl/abbr.pkl', 'rb') as infile:\n",
    "       abbr = pickle.load(infile)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Clean Text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# remove special characters\n",
    "new_resume = re.sub(r'([-/*&%+();?.,@#$^])', ' ', new_resume)\n",
    "\n",
    "cleaner = ['\\xa0']\n",
    "for c in cleaner:\n",
    "    new_resume = new_resume.replace(c,'')\n",
    "\n",
    "# remove multiple white spaces\n",
    "new_resume = re.sub('\\n',' ', new_resume)\n",
    "new_resume = re.sub('\\t',' ', new_resume)\n",
    "new_resume = re.sub(r'''[^0-9a-zA-Z ]+''', '', new_resume)\n",
    "new_resume = re.sub(' +',' ', new_resume)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'BRYANT J BIGGS 3134 Avalon Way Bloomingdale New Jersey bryantbiggs gmail com EDUCATION Purdue University West Lafayette IN Bachelor of Science M E T Received December 19 2009 Program GPA 3 28 4 00 Arizona State University Tempe AZ Bachelor of Science Software Eng Expected February 2017 Program GPA 4 00 4 00 EXPERIENCE DATA SCIENTIST Metis New York City NY June 2016 Current An immersive 12 week data science boot camp to learn and apply concepts such as data science data mining supervised SVM decision trees random forests KNN and unsupervised NLP KMeans machine learning statistical inference Bayesian statistics regression analysis data visualization and cloud computing Technologies used taught Python Scikit learn SciPy NumPy Pandas BeautifulSoup Selenium Matplotlib Flask Django Javascript D3 js jQuery SQL and NoSQL databases PostgreSQL MongoDB Apache Hadoop MapReduce and Spark APPLICATION ENGINEER Saint Gobain Performance Plastics Wayne NJ June 2014 Current Product design for PTFE based plain bearings used primarily in automotive and high performance bicycle applications New product application development from initial prototype through launch within a global engineering team Development of enterprise applications to assist in rapid engineering drawing document creation automated script to create design based recommendation document for customers material usage calculator Inventor based script to automate detail drawings and macro based 3D parametric models to create internal and external tool sets Maintained multiple Access database used in project tracking Designed and created new database used by North American HR team to review employee performance metrics and career growth PRODUCT ENGINEER Wyman Gordon Grafton MA December 2012 May 2014 Three dimensional design of forging process models forging dies heat treatment and machining fixtures Finite elemental analysis FEA of material flow cavity fill defect analysis tool stress and dead zone mitigation Continuous cost reduction efforts with emphasis on material utilization and reduction in operational steps Process and product documentation improvement work instructions and master router Led new product introduction to educate facility on new products DESIGN ENGINEER Stalcop U S L P Thorntown IN March 2010 November 2012 Tool design research and development for cold forming ferrous and non ferrous metals Process design for high and low volume manufacturing as well as prototype development Finite elemental analysis FEA of material flow and tool stress using Deform 2D 3D F3 Quality documentation including PPAP documentation PFMEA capability studies gage R R process control plan process inspection flow diagram and sample inspection reports SOFTWARE PROFICIENCIES Microsoft Office SolidWorks Autodesk Inventor Visual Studio VBA Python SQL Machine Learning MongoDB PostgreSQL NumPy SciPy Github'"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "new_resume"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Remove StopWords"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# cache stopwords first to reduce compute time\n",
    "cachedStopWords = stopwords.words(\"english\")\n",
    "cachedStopWords += ['tot']\n",
    "cachedStopWords += cities\n",
    "cachedStopWords += abbr\n",
    "cachedStopWords = list(set(cachedStopWords))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "300\n"
     ]
    }
   ],
   "source": [
    "# convert all text to lower case and separate into list\n",
    "res_text = [x for x in new_resume.lower().split()]\n",
    "\n",
    "# remove stopwords\n",
    "res_text = ' '.join([item for item in res_text if item not in cachedStopWords])\n",
    "print(len(res_text.split()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'j 3134 bryantbiggs gmail com education bachelor e received december 19 2009 program gpa 3 28 4 00 bachelor software eng expected february 2017 program gpa 4 00 4 00 experience data scientist metis 2016 current immersive 12 week data boot learn apply concepts data data mining supervised svm decision trees forests knn unsupervised nlp kmeans machine learning statistical inference bayesian statistics regression analysis data visualization computing technologies used taught python scikit learn scipy numpy pandas beautifulsoup selenium matplotlib flask django javascript d3 js jquery sql nosql databases postgresql mongodb hadoop mapreduce spark application engineer gobain performance plastics nj 2014 current product ptfe based bearings used primarily automotive performance bicycle applications product application development initial prototype launch within global engineering team development applications assist engineering drawing document creation automated script create based recommendation document customers material usage calculator inventor based script automate detail drawings macro based 3d parametric models create internal external tool sets maintained multiple access database used project tracking designed created database used hr team review employee performance metrics career growth product engineer wyman december 2012 2014 dimensional forging process models forging dies heat treatment machining fixtures finite elemental analysis fea material flow cavity fill defect analysis tool stress dead zone mitigation continuous cost reduction efforts emphasis material utilization reduction operational steps process product documentation improvement work instructions master router led product introduction educate facility products engineer stalcop u l p 2010 november 2012 tool development forming ferrous non ferrous metals process volume manufacturing well prototype development finite elemental analysis fea material flow tool stress using deform 2d 3d f3 quality documentation including ppap documentation pfmea capability studies r r process control plan process inspection flow diagram sample inspection reports software proficiencies microsoft office solidworks autodesk inventor visual vba python sql machine learning mongodb postgresql numpy scipy github'"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "res_text"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# More Cleaning (Ugly - Fix Me)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "300\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "'j 3134 bryantbiggs gmail com education bachelor e received december 19 2009 program gpa 3 28 4 00 bachelor software eng expected february 2017 program gpa 4 00 4 00 experience data scientist metis 2016 current immersive 12 week data boot learn apply concepts data data mining supervised svm decision trees forests knn unsupervised nlp kmeans machine learning statistical inference bayesian statistics regression analysis data visualization computing technologies used taught python scikit learn scipy numpy pandas beautifulsoup selenium matplotlib flask django javascript d3 js jquery sql nosql databases postgresql mongodb hadoop mapreduce spark application engineer gobain performance plastics nj 2014 current product ptfe based bearings used primarily automotive performance bicycle applications product application develop initial prototype launch within global engineer team develop applications assist engineer drawing document creation automated script create based recommendation document customers material usage calculator inventor based script automate detail drawings macro based 3d parametric models create internal external tool sets maintained multiple access database used project tracking designed created database used hr team review employee performance metrics career growth product engineer wyman december 2012 2014 dimensional forging process models forging dies heat treatment machining fixtures finite elemental analysis fea material flow cavity fill defect analysis tool stress dead zone mitigation continuous cost reduction efforts emphasis material utilization reduction operational steps process product documentation improvement work instructions master router led product introduction educate facility products engineer stalcop u l p 2010 november 2012 tool develop forming ferrous non ferrous metals process volume manufacturing well prototype develop finite elemental analysis fea material flow tool stress using deform 2d 3d f3 quality documentation including ppap documentation pfmea capability studies r r process control plan process inspection flow diagram sample inspection report software proficiencies microsoft office solidworks autodesk inventor visual vba python sql machine learning mongodb postgresql numpy scipy github'"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "res_text = res_text.replace('aacademic', 'academic')\n",
    "res_text = res_text.replace('aaccounts', 'account')\n",
    "res_text = res_text.replace('aaerobic', 'aerobic')\n",
    "res_text = res_text.replace('aademy', 'academy')\n",
    "\n",
    "res_text = res_text.replace('aanalyst', 'analyst')\n",
    "res_text = res_text.replace('aanalyzed', 'analyst')\n",
    "\n",
    "res_text = res_text.replace('reports', 'report')\n",
    "res_text = res_text.replace('reporting', 'report')\n",
    "\n",
    "res_text = res_text.replace('engineering', 'engineer')\n",
    "res_text = res_text.replace('engineers', 'engineer')\n",
    "\n",
    "res_text = res_text.replace('services', 'service')\n",
    "res_text = res_text.replace('servicing', 'service')\n",
    "\n",
    "res_text = res_text.replace('systems', 'system')\n",
    "\n",
    "res_text = res_text.replace('abandoned', 'abandon')\n",
    "res_text = res_text.replace('abandoner', 'abandon')\n",
    "res_text = res_text.replace('abandoning', 'abandon')\n",
    "res_text = res_text.replace('abandonment', 'abandon')\n",
    "res_text = res_text.replace('abandons', 'abandon')\n",
    "\n",
    "res_text = res_text.replace('management', 'manage')\n",
    "res_text = res_text.replace('manager', 'manage')\n",
    "\n",
    "res_text = res_text.replace('development', 'develop')\n",
    "res_text = res_text.replace('developer', 'develop')\n",
    "\n",
    "print(len(res_text.split()))\n",
    "res_text"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Word Counts"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def get_wordcount(text_list, min_ct=3, most_common=30, get_all=False):\n",
    "    '''\n",
    "    returns => most common\n",
    "    '''\n",
    "    # get wordcount counter object\n",
    "    word_count = Counter(text_list)\n",
    "\n",
    "    # remove words that occur min_ct times or less\n",
    "    word_count = Counter({k:v for k, v in word_count.items() if v >= min_ct})\n",
    "\n",
    "    if get_all:\n",
    "        # return all\n",
    "        word_count = word_count.items()\n",
    "    else:\n",
    "        # limit wordcounts for visualization\n",
    "        word_count = word_count.most_common(most_common)\n",
    "    \n",
    "    return word_count"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Extrack Noun Phrases"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "WordList(['bryantbiggs gmail com education bachelor e', 'program gpa', 'bachelor software eng', 'program gpa', 'experience data scientist metis', 'current immersive', 'week data boot', 'concepts data data', 'svm decision trees forests knn', 'nlp kmeans machine learning', 'statistical inference bayesian statistics regression analysis data visualization', 'python scikit', 'scipy numpy pandas beautifulsoup selenium matplotlib flask django javascript d3 js jquery sql nosql databases postgresql mongodb hadoop mapreduce', 'application engineer gobain performance plastics nj', 'current product ptfe', 'automotive performance bicycle applications product application', 'initial prototype launch', 'global engineer team', 'engineer drawing document creation', 'recommendation document customers material usage calculator inventor', 'script automate detail drawings macro', '3d parametric models', 'internal external tool', 'multiple access database', 'hr team review employee performance metrics career growth product engineer wyman december', 'process models', 'heat treatment', 'fixtures finite elemental analysis fea material flow cavity', 'defect analysis tool stress', 'dead zone mitigation', 'continuous cost reduction efforts emphasis material utilization reduction operational steps process product documentation improvement work instructions master router', 'product introduction educate facility products engineer stalcop u l p', 'ferrous non ferrous metals process volume', 'finite elemental analysis fea material flow tool stress', '3d f3 quality documentation', 'ppap documentation pfmea capability studies r r process control plan process inspection flow diagram sample inspection report software proficiencies microsoft office solidworks autodesk inventor visual vba python sql machine learning mongodb postgresql numpy scipy github'])"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "res_nouns = TextBlob(res_text).noun_phrases\n",
    "res_nouns"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Merge Noun Phrases Back to Text String"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'bryantbiggs gmail com education bachelor e program gpa bachelor software eng program gpa experience data scientist metis current immersive week data boot concepts data data svm decision trees forests knn nlp kmeans machine learning statistical inference bayesian statistics regression analysis data visualization python scikit scipy numpy pandas beautifulsoup selenium matplotlib flask django javascript d3 js jquery sql nosql databases postgresql mongodb hadoop mapreduce application engineer gobain performance plastics nj current product ptfe automotive performance bicycle applications product application initial prototype launch global engineer team engineer drawing document creation recommendation document customers material usage calculator inventor script automate detail drawings macro 3d parametric models internal external tool multiple access database hr team review employee performance metrics career growth product engineer wyman december process models heat treatment fixtures finite elemental analysis fea material flow cavity defect analysis tool stress dead zone mitigation continuous cost reduction efforts emphasis material utilization reduction operational steps process product documentation improvement work instructions master router product introduction educate facility products engineer stalcop u l p ferrous non ferrous metals process volume finite elemental analysis fea material flow tool stress 3d f3 quality documentation ppap documentation pfmea capability studies r r process control plan process inspection flow diagram sample inspection report software proficiencies microsoft office solidworks autodesk inventor visual vba python sql machine learning mongodb postgresql numpy scipy github '"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "temp = []\n",
    "for i in res_nouns:\n",
    "    temp.append(i)\n",
    "\n",
    "res_nouns = ''\n",
    "for x in temp:\n",
    "    res_nouns += x + ' '\n",
    "res_nouns"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "### Tokenize the documents, remove stop words and words that only appear once"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# remove common words and tokenize\n",
    "stoplist = set('for a of the and to in'.split())\n",
    "texts = [word for word in res_nouns.split()]\n",
    "\n",
    "# remove words that appear only once\n",
    "frequency = defaultdict(int)\n",
    "for text in texts:\n",
    "        frequency[text] += 1\n",
    "\n",
    "# remove words that occur less than n times\n",
    "texts = [token for token in texts if frequency[token] > 1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "str_text = ' '.join(texts)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Load Transformation Interface"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Sparse matrix LOADED as 'corpus'\n"
     ]
    }
   ],
   "source": [
    "# load sparse vector matrix\n",
    "if (os.path.exists('pkl/resume_token.mm')):\n",
    "    corpus = corpora.MmCorpus('pkl/resume_token.mm')\n",
    "    print('Sparse matrix LOADED as \\'corpus\\'')\n",
    "else:\n",
    "    print('Sparse matrix NOT FOUND')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Tokenized dictionary LOADED as 'dictionary'\n"
     ]
    }
   ],
   "source": [
    "# load tokenized dictionary\n",
    "if (os.path.exists('pkl/resume_token.dict')):\n",
    "    dictionary = corpora.Dictionary.load('pkl/resume_token.dict')\n",
    "    print('Tokenized dictionary LOADED as \\'dictionary\\'')\n",
    "else:\n",
    "    print('Tokenized dictionary NOT FOUND')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Load Trained Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "lsi_mdl = models.LsiModel.load('pkl/lsi_mdl.lsi')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[(0, 1.1400808987070936), (1, 0.66462049853676375), (2, -0.086946665471172441), (3, -0.20132470490451279), (4, -0.057053573572383075), (5, -0.15715645736246531), (6, -0.053012905460766535), (7, 0.085334837310623649), (8, 0.20384899948247764), (9, -0.094462004934014243), (10, 0.33054847773367119), (11, 0.33293221685077229), (12, -0.20589040353110491), (13, 0.0022485856318101823), (14, 0.21482950705197201), (15, 0.12650552541640864), (16, -0.075179024473273307), (17, 0.054494555422042207), (18, -0.072124052126883609), (19, 0.10908108256295219), (20, 0.053431851560994788), (21, 0.077374581814657784), (22, -0.28209884396980905), (23, 0.053128462536337302), (24, 0.19514019183231066), (25, -0.13951986929400334), (26, 0.26026167235649178), (27, -0.1806338176541899), (28, -0.3651136949128761), (29, 0.12832464161078413), (30, 0.28239321318959942), (31, 0.041740281442491797), (32, -0.45030988505653258), (33, 0.14682592905643618), (34, -0.11707318057360952), (35, 0.16515846290020197), (36, 0.07624908976693659), (37, -0.10953036594756015), (38, -0.089074216512978796), (39, 0.025742787078256914), (40, 0.057305902865449475), (41, -0.17265855746441541), (42, -0.13602947751478647), (43, -0.12717481480421355), (44, 0.089665321609476212), (45, 0.27944344600458931), (46, 0.16523002017661792), (47, -0.22897538753400126), (48, -0.14910448060696291), (49, -0.49936600055563463), (50, 0.13323727410567252), (51, -0.24742124897722437), (52, 0.046175143617012034), (53, 0.068016984334376529), (54, -0.37555725284485503), (55, 0.57147681556193197), (56, 0.48756398230396514), (57, -0.25389539702190206), (58, -0.1593864539219044), (59, 0.044778089759252057), (60, 0.087415076563274741), (61, 0.16811729668592304), (62, -0.72095424324818758), (63, -0.37040163954788458), (64, -0.31210545912252785), (65, 0.041614094263390022), (66, 0.40038050619868742), (67, 0.25963090959546486), (68, 0.095125700127999105), (69, 0.26878459369833868), (70, 0.43875722734245126), (71, 0.35815148238914585), (72, -0.087354955392286535), (73, -0.59062054098831129), (74, -0.031382524056965511), (75, -0.2218987423994721), (76, 0.28402147065419719), (77, 0.5197733184412372), (78, 0.19750291166899103), (79, 0.09663673962417127), (80, -0.24501484924305936), (81, 0.19311496806968734), (82, -0.65428536339631138), (83, -0.019098789097348582), (84, 0.27595437403599199), (85, -0.18020145308040056), (86, -0.20666545338680006), (87, 0.46273419913019798), (88, 0.36867160309336106), (89, 0.19311777657524928), (90, -0.00027150792504875487), (91, 0.30771349569376449), (92, -0.10142015444450835), (93, 0.33286564584115869), (94, -0.17306071598662434), (95, 0.16980729508079631), (96, 0.36225529987821792), (97, -0.099551727102546561), (98, 0.035828953180401787), (99, -0.31849570247889852)]\n"
     ]
    }
   ],
   "source": [
    "vec_bow = dictionary.doc2bow(texts)\n",
    "vec_lsi = lsi_mdl[vec_bow] # convert the query to LSI space\n",
    "print(vec_lsi)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "index = similarities.MatrixSimilarity(lsi_mdl[corpus])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# perform a similarity query against the corpus\n",
    "sims = index[vec_lsi] \n",
    "\n",
    "# (document_number, document_similarity)\n",
    "sim_lst = list(enumerate(sims))\n",
    "\n",
    "# sort relevant documents first\n",
    "sim_lst.sort(key=operator.itemgetter(1), reverse=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(14945, 0.80436462),\n",
       " (8007, 0.80401808),\n",
       " (9004, 0.80005157),\n",
       " (13984, 0.79582334),\n",
       " (3344, 0.77749413)]"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sim_lst[1:6]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'schlenker data scientist course director logit data detail data scientist experience r python current statistics machine learning packages previous experience mathematics engineer insurance value data analysis data scientist course director logit data januari direct class students course week bootcamp learning algorithms data weeks lecture material linear algebra probability statistics important methods machine learning students examples python code independent data projects data scientist nyc data september r python current algorithms software packages data scientists beautiful soup analyze data b h photo video website shiny app visualize data analysis scorecard dataset teaching assistant augustus semester calculus traditional classroom learning mathematica software curriculum mathematica calculus lab exercises mathematics mei bachelor mathematics juni vaardigheden python numpy scipy pandas matlab rstudio sql scikit linux matlab matplotlib java mathematica aanvullende informatie technical skills software r python sql git linux ggplot shiny beautiful soup numpy scipy matplotlib pandas seaborn scikit c c java android sdk matlab mathematica coursework simple multiple linear regression logistic regression time series analysis web naive bayes linear discriminant analysis support vector machines gradient descent computational linear algebra statistics probability stochastic processes graph theory optimization linear'"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.read_pickle('pkl/df_stop_noun.pkl')\n",
    "df.iloc[14945]['resume_nouns']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}