make_data_dir.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import os\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "\n",
    "from glob import glob"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>cancer</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0015ceb851d7251b8f399e39779d1e7d</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0030a160d58723ff36d73f41b170ec21</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>003f41c78e6acfa92430a057ac0b306e</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>006b96310a37b36cccb2ab48d10b49a3</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>008464bb8521d09a42985dd8add3d0d2</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                 id  cancer\n",
       "0  0015ceb851d7251b8f399e39779d1e7d       1\n",
       "1  0030a160d58723ff36d73f41b170ec21       0\n",
       "2  003f41c78e6acfa92430a057ac0b306e       0\n",
       "3  006b96310a37b36cccb2ab48d10b49a3       1\n",
       "4  008464bb8521d09a42985dd8add3d0d2       1"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "label_df = pd.read_csv('metadata/stage1_labels.csv')\n",
    "\n",
    "label_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/home/robert/projects/kaggle/cancer/prepd_stage1\n"
     ]
    }
   ],
   "source": [
    "%cd prepd_stage1/"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['968808e60982c76366088b4db7254671',\n",
       " 'caddd8f856b110ed9bb52872e003193f',\n",
       " 'fbae4d04285789dfa32124c86586dd09',\n",
       " '6997b392b28529c9ab67add45055fcf7',\n",
       " '645e7f46eb9b834153ecf8e2b2921fe5',\n",
       " 'a4ae73433e63549558a3d0eed9128a69',\n",
       " '9b4f0020e407fd7f980f4241fb6ac7ce',\n",
       " '487cc9003c99fa95e9a9e201d396992c',\n",
       " '7b4d476d9414c9339269efe47840d2da',\n",
       " '310b403db75226a9a44d9a29c18917b7']"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "g = glob('*.npy')\n",
    "patients = [patient.replace(\".npy\", \"\") for patient in g]\n",
    "\n",
    "patients[:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "198\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "['5d16819bd78c74448ce852a93bf423ad',\n",
       " 'a6c15206edadab0270898f03e770d730',\n",
       " 'e6d8ae8c3b0817df994a1ce3b37a7efb',\n",
       " '70f4eb8201e3155cc3e399f0ff09c5ef',\n",
       " '7daeb8ef7307849c715f7f6f3e2dd88e']"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_ids = [patient for patient in patients if patient not in label_df[\"id\"].values]\n",
    "\n",
    "print(len(test_ids))\n",
    "test_ids[:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "for id in test_ids:\n",
    "    fn = \"{}.npy\".format(id)\n",
    "    os.rename(fn, './test/' + fn)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "g = glob('*.npy')\n",
    "shuf = np.random.permutation(g)\n",
    "\n",
    "for i in range(200):\n",
    "    os.rename(shuf[i], './valid/' + shuf[i])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python [conda root]",
   "language": "python",
   "name": "conda-root-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}