datasets/Dataset-preprocessing.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "white-quick",
   "metadata": {},
   "outputs": [],
   "source": [
    "import requests\n",
    "import os\n",
    "from pathlib import Path\n",
    "import pickle\n",
    "from shutil import unpack_archive\n",
    "import numpy as np\n",
    "import io\n",
    "import matplotlib.pylab as plt\n",
    "from matplotlib.pyplot import plot, ion, show, savefig, cla, figure"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "local-bleeding",
   "metadata": {},
   "outputs": [],
   "source": [
    "urls = dict()\n",
    "urls['ecg']=['http://www.cs.ucr.edu/~eamonn/discords/ECG_data.zip',\n",
    "             'http://www.cs.ucr.edu/~eamonn/discords/mitdbx_mitdbx_108.txt',\n",
    "             'http://www.cs.ucr.edu/~eamonn/discords/qtdbsele0606.txt',\n",
    "             'http://www.cs.ucr.edu/~eamonn/discords/chfdbchf15.txt',\n",
    "             'http://www.cs.ucr.edu/~eamonn/discords/qtdbsel102.txt']\n",
    "urls['gesture']=['http://www.cs.ucr.edu/~eamonn/discords/ann_gun_CentroidA']\n",
    "urls['space_shuttle']=['http://www.cs.ucr.edu/~eamonn/discords/TEK16.txt',\n",
    "                       'http://www.cs.ucr.edu/~eamonn/discords/TEK17.txt',\n",
    "                       'http://www.cs.ucr.edu/~eamonn/discords/TEK14.txt']\n",
    "urls['respiration']=['http://www.cs.ucr.edu/~eamonn/discords/nprs44.txt',\n",
    "                     'http://www.cs.ucr.edu/~eamonn/discords/nprs43.txt']\n",
    "urls['power_demand']=['http://www.cs.ucr.edu/~eamonn/discords/power_data.txt']\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "likely-gossip",
   "metadata": {},
   "outputs": [],
   "source": [
    "def download_dataset(urls):\n",
    "    for dataname in urls:\n",
    "        raw_dir = Path('datasets', dataname, 'raw')\n",
    "        raw_dir.mkdir(parents=True, exist_ok=True)\n",
    "        for url in urls[dataname]:\n",
    "            filename = raw_dir.joinpath(Path(url).name)\n",
    "            print('Downloading', url)\n",
    "            resp =requests.get(url)\n",
    "            filename.write_bytes(resp.content)\n",
    "            if filename.suffix=='':\n",
    "                filename.rename(filename.with_suffix('.txt'))\n",
    "            print('Saving to', filename.with_suffix('.txt'))\n",
    "            if filename.suffix=='.zip':\n",
    "                print('Extracting to', filename)\n",
    "                unpack_archive(str(filename), extract_dir=str(raw_dir))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "short-screw",
   "metadata": {},
   "outputs": [],
   "source": [
    "#download_dataset(urls)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "funded-salvation",
   "metadata": {},
   "outputs": [],
   "source": [
    "# this function load one .cvs (a sequence)\n",
    "def load_data_ori(dataset, dataset_folder='datasets'):\n",
    "    raw_dir = Path('datasets', dataset, 'raw')\n",
    "    readings = {}\n",
    "    idx_anomaly = {}\n",
    "    t = {}\n",
    "    t_unit = {}\n",
    "    for filepath in raw_dir.glob('*.txt'):\n",
    "        with open(str(filepath)) as f:\n",
    "            #f = open(str(filepath))\n",
    "            #print(\"shape of f:\", np.array(f.read()).shape)\n",
    "            for i, line in enumerate(f):\n",
    "                tokens = []#[float(token) for token in line.split()]\n",
    "                if raw_dir.parent.name == 'ecg':\n",
    "                    tokens.pop(0)\n",
    "                if filepath.name == 'chfdbchf15.txt':\n",
    "                    tokens.append(1.0) if 2250 < i < 2400 else tokens.append(0.0)\n",
    "                elif filepath.name == 'xmitdb_x108_0.txt':\n",
    "                    tokens.append(1.0) if 4020 < i < 4400 else tokens.append(0.0)\n",
    "                elif filepath.name == 'mitdb__100_180.txt':\n",
    "                    tokens.append(1.0) if 1800 < i < 1990 else tokens.append(0.0)\n",
    "                elif filepath.name == 'chfdb_chf01_275.txt':\n",
    "                    tokens.append(1.0) if 2330 < i < 2600 else tokens.append(0.0)\n",
    "                elif filepath.name == 'ltstdb_20221_43.txt':\n",
    "                    tokens.append(1.0) if 650 < i < 780 else tokens.append(0.0)\n",
    "                elif filepath.name == 'ltstdb_20321_240.txt':\n",
    "                    tokens.append(1.0) if 710 < i < 850 else tokens.append(0.0)\n",
    "                elif filepath.name == 'chfdb_chf13_45590.txt':\n",
    "                    tokens.append(1.0) if 2800 < i < 2960 else tokens.append(0.0)\n",
    "                elif filepath.name == 'stdb_308_0.txt':\n",
    "                    tokens.append(1.0) if 2290 < i < 2550 else tokens.append(0.0)\n",
    "                elif filepath.name == 'qtdbsel102.txt':\n",
    "                    tokens.append(1.0) if 4230 < i < 4430 else tokens.append(0.0)\n",
    "                elif filepath.name == 'ann_gun_CentroidA.txt':\n",
    "                    tokens.append(1.0) if 2070 < i < 2810 else tokens.append(0.0)\n",
    "                elif filepath.name == 'TEK16.txt':\n",
    "                    tokens.append(1.0) if 4270 < i < 4370 else tokens.append(0.0)\n",
    "                elif filepath.name == 'TEK17.txt':\n",
    "                    tokens.append(1.0) if 2100 < i < 2145 else tokens.append(0.0)\n",
    "                elif filepath.name == 'TEK14.txt':\n",
    "                    tokens.append(1.0) if 1100 < i < 1200 or 1455 < i < 1955 else tokens.append(0.0)\n",
    "                elif filepath.name == 'nprs44.txt':\n",
    "                    tokens.append(1.0) if 16192 < i < 16638 or 20457 < i < 20911 else tokens.append(0.0)\n",
    "                elif filepath.name == 'nprs43.txt':\n",
    "                    tokens.append(1.0) if 12929 < i < 13432 or 14877 < i < 15086 or 15729 < i < 15924 else tokens.append(0.0)\n",
    "                elif filepath.name == 'power_data.txt':\n",
    "                    tokens.append(1.0) if 8254 < i < 8998 or 11348 < i < 12143 or 33883 < i < 34601 else tokens.append(0.0)\n",
    "                try:\n",
    "                    readings[filepath.name].append(line)\n",
    "                    if tokens[-1] == 1.0:\n",
    "                        idx_anomaly[filepath.name].append(i)\n",
    "                except:\n",
    "                    readings[filepath.name] = [tokens[:-1]]\n",
    "                    if tokens[-1] == 1.0:\n",
    "                        idx_anomaly[filepath.name] = [i]#indent below\n",
    "            print(\"readings of\",filepath.name, \"shape:\",np.shape(readings[filepath.name]))        \n",
    "            readings[filepath.name] = np.asarray(readings[filepath.name])\n",
    "            idx_anomaly[filepath.name] = np.asarray(idx_anomaly[filepath.name])\n",
    "            t_unit[filepath.name] = 'time_unit'\n",
    "            t[filepath.name] = np.array(range(readings[filepath.name].shape[0]))\n",
    "    return t, t_unit, idx_anomaly, readings\n",
    "#     if dataset == 'ambient_temp':\n",
    "#         data_file = os.path.join(csv_folder, 'ambient_temperature_system_failure.csv')\n",
    "#         anomalies = ['2013-12-22 20:00:00', '2014-04-13 09:00:00']\n",
    "#         t_unit = 'hour'\n",
    "#     elif dataset == 'cpu_utilization':\n",
    "#         data_file = os.path.join(csv_folder, 'cpu_utilization_asg_misconfiguration.csv')\n",
    "#         anomalies = ['2014-07-12 02:04:00', '2014-07-14 21:44:00']\n",
    "#         t_unit = '5 min'\n",
    "#     elif dataset == 'ec2_request':\n",
    "#         data_file = os.path.join(csv_folder, 'ec2_request_latency_system_failure.csv')\n",
    "#         anomalies = ['2014-03-14 09:06:00', '2014-03-18 22:41:00', '2014-03-21 03:01:00']\n",
    "#         t_unit = '5 min'\n",
    "#     elif dataset == 'machine_temp':\n",
    "#         data_file = os.path.join(csv_folder, 'machine_temperature_system_failure.csv')\n",
    "#         anomalies = ['2013-12-11 06:00:00', '2013-12-16 17:25:00', '2014-01-28 13:55:00', '2014-02-08 14:30:00']\n",
    "#         t_unit = '5 min'\n",
    "#     elif dataset == 'rogue_agent_key_hold':\n",
    "#         data_file = os.path.join(csv_folder, 'rogue_agent_key_hold.csv')\n",
    "#         anomalies = ['2014-07-15 08:30:00', '2014-07-17 09:50:00']\n",
    "#         t_unit = '5 min'\n",
    "#     elif dataset == 'rogue_agent_key_updown':\n",
    "#         data_file = os.path.join(csv_folder, 'rogue_agent_key_updown.csv')\n",
    "#         anomalies = ['2014-07-15 04:00:00', '2014-07-17 08:50:00']\n",
    "#         t_unit = '5 min'\n",
    "#     elif dataset == 'nyc_taxi':\n",
    "#         data_file = os.path.join(csv_folder, 'nyc_taxi.csv')\n",
    "#         anomalies = ['2014-11-01 19:00:00', '2014-11-27 15:30:00', '2014-12-25 15:00:00', '2015-01-01 01:00:00', \n",
    "#                      '2015-01-27 00:00:00']\n",
    "#         t_unit = '30 min'\n",
    "    \n",
    "#     t = []\n",
    "#     readings = []\n",
    "#     idx_anomaly = []\n",
    "#     i = 0\n",
    "#     with open(data_file) as csvfile:\n",
    "#         readCSV = csv.reader(csvfile, delimiter=',')\n",
    "#         print(\"\\n--> Anomalies occur at:\")\n",
    "#         for row in readCSV:\n",
    "#             if i > 0:\n",
    "#                 t.append(i)\n",
    "#                 readings.append(float(row[1]))\n",
    "#                 for j in range(len(anomalies)):\n",
    "#                     if row[0] == anomalies[j]:\n",
    "#                         idx_anomaly.append(i)\n",
    "#                         print(\"  timestamp #{}: {}\".format(j, row[0]))\n",
    "#             i = i + 1\n",
    "#     t = np.asarray(t)\n",
    "#     readings = np.asarray(readings)\n",
    "#     print(\"\\nOriginal csv file contains {} timestamps.\".format(t.shape))\n",
    "#     print(\"Processed time series contain {} readings.\".format(readings.shape))\n",
    "#     print(\"Anomaly indices are {}\".format(idx_anomaly))\n",
    "    \n",
    "#     return t, t_unit, readings, idx_anomaly"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "sitting-catering",
   "metadata": {},
   "outputs": [],
   "source": [
    "def load_data(dataset, dataset_folder='datasets'):\n",
    "    raw_dir = Path('datasets', dataset, 'raw')\n",
    "    readings = {}\n",
    "    idx_anomaly = {}\n",
    "    t = {}\n",
    "    t_unit = {}\n",
    "    for filepath in raw_dir.glob('*.txt'):\n",
    "            f = np.loadtxt(str(filepath))\n",
    "            if raw_dir.parent.name == 'ecg':\n",
    "                f = f[1:]\n",
    "            print(f.shape)\n",
    "            readings[filepath.name] = f\n",
    "            #f = open(str(filepath))\n",
    "            #print(\"shape of f:\", np.array(f.read()).shape)\n",
    "            for i in range(f.shape[0]):\n",
    "                if filepath.name == 'chfdbchf15.txt':\n",
    "                    #if 2250 < i < 2400: \n",
    "                    idx_anomaly[filepath.name] = np.array(range(2251,2400))\n",
    "                elif filepath.name == 'xmitdb_x108_0.txt':\n",
    "                    #if 4020 < i < 4400:\n",
    "                    idx_anomaly[filepath.name] = np.array(range(4021,4400)) \n",
    "                elif filepath.name == 'mitdb__100_180.txt':\n",
    "                    #if 1800 < i < 1990:\n",
    "                    idx_anomaly[filepath.name] = np.array(range(1801,1990)) \n",
    "                elif filepath.name == 'chfdb_chf01_275.txt':\n",
    "                    #if 2330 < i < 2600:\n",
    "                    idx_anomaly[filepath.name] = np.array(range(2331,2600))  \n",
    "                elif filepath.name == 'ltstdb_20221_43.txt':\n",
    "                    #if 650 < i < 780: \n",
    "                    idx_anomaly[filepath.name] = np.array(range(651,780)) \n",
    "                elif filepath.name == 'ltstdb_20321_240.txt':\n",
    "                    #if 710 < i < 850: \n",
    "                    idx_anomaly[filepath.name] = np.array(range(711,850)) \n",
    "                elif filepath.name == 'chfdb_chf13_45590.txt':\n",
    "                    #if 2800 < i < 2960: \n",
    "                    idx_anomaly[filepath.name] = np.array(range(2801,2960)) \n",
    "                elif filepath.name == 'stdb_308_0.txt':\n",
    "                    #if 2290 < i < 2550: \n",
    "                    idx_anomaly[filepath.name] = np.array(range(2291,2550))\n",
    "                elif filepath.name == 'qtdbsel102.txt':\n",
    "                    #if 4230 < i < 4430:\n",
    "                    idx_anomaly[filepath.name] = np.array(range(4231,4430))   \n",
    "                elif filepath.name == 'ann_gun_CentroidA.txt':\n",
    "                    #if 2070 < i < 2810:\n",
    "                    idx_anomaly[filepath.name] = np.array(range(2071,2810))  \n",
    "                elif filepath.name == 'TEK16.txt':\n",
    "                    #if 4270 < i < 4370: \n",
    "                    idx_anomaly[filepath.name] = np.array(range(4271,4370)) \n",
    "                elif filepath.name == 'TEK17.txt':\n",
    "                    #if 2100 < i < 2145: \n",
    "                    idx_anomaly[filepath.name] = np.array(range(2101,2145)) \n",
    "                elif filepath.name == 'TEK14.txt':\n",
    "                    #if 1100 < i < 1200 or 1455 < i < 1955:\n",
    "                    idx_anomaly[filepath.name] = np.append(np.array(range(1101,1200)),np.array(range(1456,1955))) \n",
    "                elif filepath.name == 'nprs44.txt':\n",
    "                    #if 16192 < i < 16638 or 20457 < i < 20911:\n",
    "                    idx_anomaly[filepath.name] = np.append(np.array(range(16193,16638)),np.array(range(20458,20911)))   \n",
    "                elif filepath.name == 'nprs43.txt':\n",
    "                    #if 12929 < i < 13432 or 14877 < i < 15086 or 15729 < i < 15924:\n",
    "                    idx_anomaly[filepath.name] = np.append(np.array(range(12930,13432)),np.append(np.array(range(14878,15086)),np.array(range(15730,15924))))\n",
    "                elif filepath.name == 'power_data.txt':\n",
    "                    #if 8254 < i < 8998 or 11348 < i < 12143 or 33883 < i < 34601:\n",
    "                    idx_anomaly[filepath.name] = np.append(np.array(range(8255,8998)),np.append(np.array(range(11349,12143)),np.array(range(33884,34601))))   \n",
    "\n",
    "            #print(\"readings of\",filepath.name, \"shape:\",np.shape(readings[filepath.name]))        \n",
    "            readings[filepath.name] = np.asarray(readings[filepath.name])\n",
    "            idx_anomaly[filepath.name] = np.asarray(idx_anomaly[filepath.name])\n",
    "            t_unit[filepath.name] = 'time_unit'\n",
    "            t[filepath.name] = np.array(range(readings[filepath.name].shape[0]))\n",
    "    return t, t_unit, idx_anomaly, readings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "passing-light",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 8255,  8256,  8257, ..., 34598, 34599, 34600])"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.append(np.array(range(8255,8998)),np.append(np.array(range(11349,12143)),np.array(range(33884,34601))))   "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "helpful-novelty",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(5000,)\n",
      "(5000,)\n",
      "(5000,)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "({'TEK14.txt': array([   0,    1,    2, ..., 4997, 4998, 4999]),\n",
       "  'TEK16.txt': array([   0,    1,    2, ..., 4997, 4998, 4999]),\n",
       "  'TEK17.txt': array([   0,    1,    2, ..., 4997, 4998, 4999])},\n",
       " {'TEK14.txt': 'time_unit',\n",
       "  'TEK16.txt': 'time_unit',\n",
       "  'TEK17.txt': 'time_unit'},\n",
       " {'TEK14.txt': array([1101, 1102, 1103, 1104, 1105, 1106, 1107, 1108, 1109, 1110, 1111,\n",
       "         1112, 1113, 1114, 1115, 1116, 1117, 1118, 1119, 1120, 1121, 1122,\n",
       "         1123, 1124, 1125, 1126, 1127, 1128, 1129, 1130, 1131, 1132, 1133,\n",
       "         1134, 1135, 1136, 1137, 1138, 1139, 1140, 1141, 1142, 1143, 1144,\n",
       "         1145, 1146, 1147, 1148, 1149, 1150, 1151, 1152, 1153, 1154, 1155,\n",
       "         1156, 1157, 1158, 1159, 1160, 1161, 1162, 1163, 1164, 1165, 1166,\n",
       "         1167, 1168, 1169, 1170, 1171, 1172, 1173, 1174, 1175, 1176, 1177,\n",
       "         1178, 1179, 1180, 1181, 1182, 1183, 1184, 1185, 1186, 1187, 1188,\n",
       "         1189, 1190, 1191, 1192, 1193, 1194, 1195, 1196, 1197, 1198, 1199,\n",
       "         1456, 1457, 1458, 1459, 1460, 1461, 1462, 1463, 1464, 1465, 1466,\n",
       "         1467, 1468, 1469, 1470, 1471, 1472, 1473, 1474, 1475, 1476, 1477,\n",
       "         1478, 1479, 1480, 1481, 1482, 1483, 1484, 1485, 1486, 1487, 1488,\n",
       "         1489, 1490, 1491, 1492, 1493, 1494, 1495, 1496, 1497, 1498, 1499,\n",
       "         1500, 1501, 1502, 1503, 1504, 1505, 1506, 1507, 1508, 1509, 1510,\n",
       "         1511, 1512, 1513, 1514, 1515, 1516, 1517, 1518, 1519, 1520, 1521,\n",
       "         1522, 1523, 1524, 1525, 1526, 1527, 1528, 1529, 1530, 1531, 1532,\n",
       "         1533, 1534, 1535, 1536, 1537, 1538, 1539, 1540, 1541, 1542, 1543,\n",
       "         1544, 1545, 1546, 1547, 1548, 1549, 1550, 1551, 1552, 1553, 1554,\n",
       "         1555, 1556, 1557, 1558, 1559, 1560, 1561, 1562, 1563, 1564, 1565,\n",
       "         1566, 1567, 1568, 1569, 1570, 1571, 1572, 1573, 1574, 1575, 1576,\n",
       "         1577, 1578, 1579, 1580, 1581, 1582, 1583, 1584, 1585, 1586, 1587,\n",
       "         1588, 1589, 1590, 1591, 1592, 1593, 1594, 1595, 1596, 1597, 1598,\n",
       "         1599, 1600, 1601, 1602, 1603, 1604, 1605, 1606, 1607, 1608, 1609,\n",
       "         1610, 1611, 1612, 1613, 1614, 1615, 1616, 1617, 1618, 1619, 1620,\n",
       "         1621, 1622, 1623, 1624, 1625, 1626, 1627, 1628, 1629, 1630, 1631,\n",
       "         1632, 1633, 1634, 1635, 1636, 1637, 1638, 1639, 1640, 1641, 1642,\n",
       "         1643, 1644, 1645, 1646, 1647, 1648, 1649, 1650, 1651, 1652, 1653,\n",
       "         1654, 1655, 1656, 1657, 1658, 1659, 1660, 1661, 1662, 1663, 1664,\n",
       "         1665, 1666, 1667, 1668, 1669, 1670, 1671, 1672, 1673, 1674, 1675,\n",
       "         1676, 1677, 1678, 1679, 1680, 1681, 1682, 1683, 1684, 1685, 1686,\n",
       "         1687, 1688, 1689, 1690, 1691, 1692, 1693, 1694, 1695, 1696, 1697,\n",
       "         1698, 1699, 1700, 1701, 1702, 1703, 1704, 1705, 1706, 1707, 1708,\n",
       "         1709, 1710, 1711, 1712, 1713, 1714, 1715, 1716, 1717, 1718, 1719,\n",
       "         1720, 1721, 1722, 1723, 1724, 1725, 1726, 1727, 1728, 1729, 1730,\n",
       "         1731, 1732, 1733, 1734, 1735, 1736, 1737, 1738, 1739, 1740, 1741,\n",
       "         1742, 1743, 1744, 1745, 1746, 1747, 1748, 1749, 1750, 1751, 1752,\n",
       "         1753, 1754, 1755, 1756, 1757, 1758, 1759, 1760, 1761, 1762, 1763,\n",
       "         1764, 1765, 1766, 1767, 1768, 1769, 1770, 1771, 1772, 1773, 1774,\n",
       "         1775, 1776, 1777, 1778, 1779, 1780, 1781, 1782, 1783, 1784, 1785,\n",
       "         1786, 1787, 1788, 1789, 1790, 1791, 1792, 1793, 1794, 1795, 1796,\n",
       "         1797, 1798, 1799, 1800, 1801, 1802, 1803, 1804, 1805, 1806, 1807,\n",
       "         1808, 1809, 1810, 1811, 1812, 1813, 1814, 1815, 1816, 1817, 1818,\n",
       "         1819, 1820, 1821, 1822, 1823, 1824, 1825, 1826, 1827, 1828, 1829,\n",
       "         1830, 1831, 1832, 1833, 1834, 1835, 1836, 1837, 1838, 1839, 1840,\n",
       "         1841, 1842, 1843, 1844, 1845, 1846, 1847, 1848, 1849, 1850, 1851,\n",
       "         1852, 1853, 1854, 1855, 1856, 1857, 1858, 1859, 1860, 1861, 1862,\n",
       "         1863, 1864, 1865, 1866, 1867, 1868, 1869, 1870, 1871, 1872, 1873,\n",
       "         1874, 1875, 1876, 1877, 1878, 1879, 1880, 1881, 1882, 1883, 1884,\n",
       "         1885, 1886, 1887, 1888, 1889, 1890, 1891, 1892, 1893, 1894, 1895,\n",
       "         1896, 1897, 1898, 1899, 1900, 1901, 1902, 1903, 1904, 1905, 1906,\n",
       "         1907, 1908, 1909, 1910, 1911, 1912, 1913, 1914, 1915, 1916, 1917,\n",
       "         1918, 1919, 1920, 1921, 1922, 1923, 1924, 1925, 1926, 1927, 1928,\n",
       "         1929, 1930, 1931, 1932, 1933, 1934, 1935, 1936, 1937, 1938, 1939,\n",
       "         1940, 1941, 1942, 1943, 1944, 1945, 1946, 1947, 1948, 1949, 1950,\n",
       "         1951, 1952, 1953, 1954]),\n",
       "  'TEK16.txt': array([4271, 4272, 4273, 4274, 4275, 4276, 4277, 4278, 4279, 4280, 4281,\n",
       "         4282, 4283, 4284, 4285, 4286, 4287, 4288, 4289, 4290, 4291, 4292,\n",
       "         4293, 4294, 4295, 4296, 4297, 4298, 4299, 4300, 4301, 4302, 4303,\n",
       "         4304, 4305, 4306, 4307, 4308, 4309, 4310, 4311, 4312, 4313, 4314,\n",
       "         4315, 4316, 4317, 4318, 4319, 4320, 4321, 4322, 4323, 4324, 4325,\n",
       "         4326, 4327, 4328, 4329, 4330, 4331, 4332, 4333, 4334, 4335, 4336,\n",
       "         4337, 4338, 4339, 4340, 4341, 4342, 4343, 4344, 4345, 4346, 4347,\n",
       "         4348, 4349, 4350, 4351, 4352, 4353, 4354, 4355, 4356, 4357, 4358,\n",
       "         4359, 4360, 4361, 4362, 4363, 4364, 4365, 4366, 4367, 4368, 4369]),\n",
       "  'TEK17.txt': array([2101, 2102, 2103, 2104, 2105, 2106, 2107, 2108, 2109, 2110, 2111,\n",
       "         2112, 2113, 2114, 2115, 2116, 2117, 2118, 2119, 2120, 2121, 2122,\n",
       "         2123, 2124, 2125, 2126, 2127, 2128, 2129, 2130, 2131, 2132, 2133,\n",
       "         2134, 2135, 2136, 2137, 2138, 2139, 2140, 2141, 2142, 2143, 2144])},\n",
       " {'TEK14.txt': array([-0.22,  0.02, -0.22, ..., -0.1 , -0.1 , -0.1 ]),\n",
       "  'TEK16.txt': array([-0.22,  0.02, -0.22, ..., -0.1 , -0.1 , -0.1 ]),\n",
       "  'TEK17.txt': array([-0.22,  0.02, -0.22, ..., -0.1 , -0.1 , -0.1 ])})"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "load_data('space_shuttle')\n",
    "#print(reads)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "clean-beginning",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "id": "perfect-newton",
   "metadata": {},
   "outputs": [],
   "source": [
    "# This function plots a dataset with the train/test split and known anomalies\n",
    "# Relies on helper function load_data()\n",
    "\n",
    "def process_and_save_specified_dataset(dataset, y_scale=5, save_file=False):\n",
    "    t, t_unit, all_idx_anomaly, all_readings = load_data(dataset)\n",
    "    all_readings_normalised = {}\n",
    "    for key in all_readings.keys():\n",
    "        readings = all_readings[key]\n",
    "        print(readings.shape)\n",
    "        idx_anomaly = all_idx_anomaly[key]\n",
    "        # split into training and test sets\n",
    "\n",
    "        idx_train = idx_test = []\n",
    "        if key == 'chfdb_chf13_45590.txt':\n",
    "            idx_train = [0, 2439]\n",
    "            idx_test = [2439, 3726]\n",
    "        elif key == 'chfdb_chf01_275.txt':\n",
    "            idx_train = [0, 1833]\n",
    "            idx_test = [1833, 3674]\n",
    "        elif key == 'chfdbchf15.txt':\n",
    "            idx_train = [3381, 14244]\n",
    "            idx_test = [33, 3381]\n",
    "        elif key == 'qtdbsel102.txt':\n",
    "            idx_train = [10093, 44828]\n",
    "            idx_test = [211, 10093]\n",
    "        elif key == 'mitdb__100_180.txt':\n",
    "            idx_train = [2328, 5271]\n",
    "            idx_test = [73, 2328]\n",
    "        elif key == 'stdb_308_0.txt':\n",
    "            idx_train = [2986, 5359]\n",
    "            idx_test = [265, 2986]\n",
    "        elif key == 'ltstdb_20321_240.txt':\n",
    "            idx_train = [1520, 3531]\n",
    "            idx_test = [73, 1520]\n",
    "        elif key == 'xmitdb_x108_0.txt':\n",
    "            idx_train = [424, 3576]\n",
    "            idx_test = [3576, 5332]\n",
    "        elif key == 'ltstdb_20221_43.txt':\n",
    "            idx_train = [1121, 3731]\n",
    "            idx_test = [0, 1121]\n",
    "        elif key == 'ann_gun_CentroidA.txt':\n",
    "            idx_train = [3000, len(readings)]\n",
    "            idx_test = [0, 3000]\n",
    "        elif key == 'nprs44.txt':\n",
    "            idx_train = [363, 12955]\n",
    "            idx_test = [12955, 24082]\n",
    "        elif key == 'nprs43.txt':\n",
    "            idx_train = [4285, 10498]\n",
    "            idx_test = [10498, 17909]\n",
    "        elif key == 'power_data.txt':\n",
    "            idx_train = [15287, 33432]\n",
    "            idx_test = [501, 15287]\n",
    "        elif key == 'TEK17.txt':\n",
    "            idx_train = [2469, 4588]\n",
    "            idx_test = [1543, 2469]\n",
    "        elif key == 'TEK16.txt':\n",
    "            idx_train = [521, 3588]\n",
    "            idx_test = [3588, 4539]\n",
    "        elif key == 'TEK14.txt':\n",
    "            idx_train = [2089, 4098]\n",
    "            idx_test = [97, 2089]\n",
    "\n",
    "        training = readings[idx_train[0]:idx_train[1]]\n",
    "        # normalise by training mean and std \n",
    "        train_m = np.mean(training, axis=0)\n",
    "        train_std = np.std(training, axis=0)\n",
    "        readings_normalised = (readings - train_m) / train_std\n",
    "        print(key, train_m, train_m.shape, train_std, train_std.shape, readings_normalised[:20])\n",
    "\n",
    "        training = readings_normalised[idx_train[0]:idx_train[1]]\n",
    "        test = readings_normalised[idx_test[0]:idx_test[1]]\n",
    "        idx_anomaly_test = idx_anomaly - idx_test[0] #+ idx_test[0] + 1?\n",
    "        #print(idx_anomaly_test)\n",
    "\n",
    "        if save_file:\n",
    "            save_dir = './datasets/{}/processed/'.format(dataset) \n",
    "            if not os.path.isdir(save_dir):\n",
    "                os.makedirs(save_dir)\n",
    "            np.savez(save_dir+key.replace('txt', 'npz'), t=t, t_unit = t_unit, readings=readings, idx_anomaly=idx_anomaly,\n",
    "                        training=training, test=test, train_m=train_m, train_std=train_std,\n",
    "                        idx_anomaly_test=idx_anomaly_test)\n",
    "            print(\"\\nProcessed time series are saved at {}\".format(save_dir+key.replace('txt', 'npz')))\n",
    "        else:\n",
    "            print(\"\\nProcessed time series are not saved.\")\n",
    "        all_readings_normalised[key] = readings_normalised\n",
    "\n",
    "         # plot the whole normalised sequence\n",
    "        fig, axs = plt.subplots(1, 1, figsize=(18, 4), edgecolor='k')\n",
    "        fig.subplots_adjust(hspace=.4, wspace=.4)\n",
    "#         # axs = axs.ravel()\n",
    "#         # for i in range(4):\n",
    "#        print(readings_normalised.shape,t.shape)\n",
    "        axs.plot(t[key], readings_normalised)\n",
    "        if idx_train[0] == 0:\n",
    "            axs.plot(idx_train[1]*np.ones(20), np.linspace(-y_scale,y_scale,20), 'b--')\n",
    "        else:\n",
    "            for i in range(2):\n",
    "                axs.plot(idx_train[i]*np.ones(20), np.linspace(-y_scale,y_scale,20), 'b--')\n",
    "        for j in range(len(idx_anomaly)):\n",
    "            axs.plot(idx_anomaly[j]*np.ones(20), np.linspace(-y_scale,y_scale,20), 'r--')\n",
    "#         #     axs.plot(data[:,1])\n",
    "        axs.grid(True)\n",
    "        axs.set_xlim(0, len(t[key]))\n",
    "        axs.set_ylim(-y_scale, y_scale)\n",
    "        axs.set_xlabel(\"timestamp (every {})\".format(t_unit[key]))\n",
    "        axs.set_ylabel(\"normalised readings\")\n",
    "        axs.set_title(\"{} dataset\\n(normalised by train mean {:.2f} and std {:.2f})\".format(dataset, np.mean(train_m), np.mean(train_std)))\n",
    "        axs.legend(('data', 'train test set split', 'anomalies'))\n",
    "\n",
    "    return all_readings_normalised"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "hungarian-customer",
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'process_and_save_specified_dataset' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-1-d393fa7ed544>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[1;32m----> 1\u001b[1;33m \u001b[0mprocess_and_save_specified_dataset\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m'space_shuttle'\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0msave_file\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;32mFalse\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[1;31mNameError\u001b[0m: name 'process_and_save_specified_dataset' is not defined"
     ]
    }
   ],
   "source": [
    "process_and_save_specified_dataset('space_shuttle', save_file=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "wireless-parent",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}