main.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "outputs": [],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "Import varie e inizializzazione delle variabili dei modelli e del tuner"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%% md\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/dati/luca/Uni-Luca/Tesi/progetto/venv/lib/python3.9/site-packages/statsmodels/tsa/base/tsa_model.py:7: FutureWarning: pandas.Int64Index is deprecated and will be removed from pandas in a future version. Use pandas.Index with the appropriate dtype instead.\n",
      "  from pandas import (to_datetime, Int64Index, DatetimeIndex, Period,\n",
      "/dati/luca/Uni-Luca/Tesi/progetto/venv/lib/python3.9/site-packages/statsmodels/tsa/base/tsa_model.py:7: FutureWarning: pandas.Float64Index is deprecated and will be removed from pandas in a future version. Use pandas.Index with the appropriate dtype instead.\n",
      "  from pandas import (to_datetime, Int64Index, DatetimeIndex, Period,\n",
      "/dati/luca/Uni-Luca/Tesi/progetto/venv/lib/python3.9/site-packages/sktime/datatypes/_series/_check.py:42: FutureWarning: pandas.Int64Index is deprecated and will be removed from pandas in a future version. Use pandas.Index with the appropriate dtype instead.\n",
      "  VALID_INDEX_TYPES = (pd.Int64Index, pd.RangeIndex, pd.PeriodIndex, pd.DatetimeIndex)\n",
      "/dati/luca/Uni-Luca/Tesi/progetto/venv/lib/python3.9/site-packages/sktime/datatypes/_panel/_check.py:45: FutureWarning: pandas.Int64Index is deprecated and will be removed from pandas in a future version. Use pandas.Index with the appropriate dtype instead.\n",
      "  VALID_INDEX_TYPES = (pd.Int64Index, pd.RangeIndex, pd.PeriodIndex, pd.DatetimeIndex)\n",
      "/dati/luca/Uni-Luca/Tesi/progetto/venv/lib/python3.9/site-packages/sktime/datatypes/_panel/_check.py:46: FutureWarning: pandas.Int64Index is deprecated and will be removed from pandas in a future version. Use pandas.Index with the appropriate dtype instead.\n",
      "  VALID_MULTIINDEX_TYPES = (pd.Int64Index, pd.RangeIndex)\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "from tensorflow import keras\n",
    "from sklearn.model_selection import train_test_split\n",
    "from IRESNs_tensorflow.time_series_datasets import *\n",
    "from IRESNs_tensorflow.models import ESN, IRESN, IIRESN\n",
    "\n",
    "PROJECT_ROOT = os.path.abspath(os.getcwd())\n",
    "DATASET_NAME = \"CharacterTrajectories\"\n",
    "DATASET_DIR = os.path.join(PROJECT_ROOT, \"datasets\")\n",
    "\n",
    "READOUT_ACTIVATION_BINARY = keras.activations.sigmoid\n",
    "LOSS_FUNCTION_BINARY = keras.losses.BinaryCrossentropy()\n",
    "\n",
    "READOUT_ACTIVATION = keras.activations.softmax\n",
    "LOSS_FUNCTION = keras.losses.SparseCategoricalCrossentropy()\n",
    "\n",
    "# Dataset dependent settings\n",
    "RESERVOIRS = 3\n",
    "OUTPUT_UNITS = 20\n",
    "\n",
    "# Tuner settings\n",
    "MAX_EPOCHS = 3\n",
    "PATIENCE = 1\n",
    "MAX_TRIALS = 2\n",
    "OVERWRITE = True\n",
    "TRIALS = 1\n",
    "\n",
    "BENCHMARKS_TRIALS = 1\n",
    "\n",
    "MINVAL = 0.01\n",
    "MAXVAL = 1.5"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "Definizione della funzione utilizzata dal tuner per istanziare i modelli per la model selection.\n",
    "Nella seguente cella si definisce un modello di tipo ESN."
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%% md\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "outputs": [],
   "source": [
    "def build_ESN(tuner):\n",
    "    ESN_model = ESN(units=100,\n",
    "                    connectivity=tuner.Float('connectivity', min_value=0.01, max_value=1.),\n",
    "                    spectral_radius=tuner.Float('spectral radius', min_value=MINVAL, max_value=MAXVAL),\n",
    "                    input_scaling=tuner.Float('input scaling', min_value=MINVAL, max_value=MAXVAL),\n",
    "                    bias_scaling=tuner.Float('bias scaling', min_value=MINVAL, max_value=MAXVAL),\n",
    "                    leaky=tuner.Float('leaky', min_value=0.01, max_value=1.0),\n",
    "                    output_units=OUTPUT_UNITS,\n",
    "                    readout_activation=READOUT_ACTIVATION,\n",
    "                    )\n",
    "\n",
    "    alpha = tuner.Float('learning rate', min_value=1e-5, max_value=1e-1, sampling='log')\n",
    "    ESN_model.compile(\n",
    "        optimizer=keras.optimizers.Adam(alpha),\n",
    "        loss=LOSS_FUNCTION,\n",
    "        metrics=['accuracy'],\n",
    "    )\n",
    "\n",
    "    return ESN_model"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "In seguito vengono definiti due tipi di modelli\n",
    "- IRESN ha #'RESERVOIRS' al suo interno di tipo ESN tutti della stessa dimensione non comunicanti.\n",
    "- IIRESN #'RESERVOIRS' al suo interno di tipo ESN tutti della stessa dimensione comunicanti."
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%% md\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "outputs": [],
   "source": [
    "def build_IRESN(tuner):\n",
    "    connectivity = [tuner.Float('connectivity ' + str(i), min_value=0.01, max_value=1.) for i in range(RESERVOIRS)]\n",
    "    normalization = [tuner.Float('spectral radius ' + str(i), min_value=MINVAL, max_value=MAXVAL) for i in\n",
    "                     range(RESERVOIRS)]\n",
    "\n",
    "    IRESN_model = IRESN(sub_reservoirs=RESERVOIRS,\n",
    "                        output_units=OUTPUT_UNITS,\n",
    "                        units=100,\n",
    "                        connectivity=connectivity,\n",
    "                        normalization=normalization,\n",
    "                        input_scaling=tuner.Float('input scaling', min_value=MINVAL, max_value=MAXVAL),\n",
    "                        bias_scaling=tuner.Float('bias_scaling', min_value=MINVAL, max_value=MAXVAL),\n",
    "                        leaky=tuner.Float('leaky', min_value=0.01, max_value=1.0),\n",
    "                        readout_activation=READOUT_ACTIVATION,\n",
    "                        )\n",
    "\n",
    "    alpha = tuner.Float('learning rate', min_value=1e-5, max_value=1e-1, sampling='log')\n",
    "    IRESN_model.compile(\n",
    "        optimizer=keras.optimizers.Adam(alpha),  # keras.optimizers.RMSprop(alpha),\n",
    "        loss=LOSS_FUNCTION,\n",
    "        metrics=['accuracy'],\n",
    "    )\n",
    "    return IRESN_model\n",
    "\n",
    "\n",
    "def build_IIRESN(tuner):\n",
    "    norm = tuner.Float('norm', min_value=MINVAL, max_value=MAXVAL)\n",
    "    normalization = [[tuner.Float('spectral radius ' + str(i), min_value=MINVAL, max_value=MAXVAL) if i == j else\n",
    "                      norm\n",
    "                      for i in range(RESERVOIRS)]\n",
    "                     for j in range(RESERVOIRS)]\n",
    "\n",
    "    ic_density = tuner.Float('connectivity X->Y', min_value=0., max_value=1.)\n",
    "    connectivity = [[tuner.Float('connectivity ' + str(i), min_value=0., max_value=1.) if i == j else\n",
    "                     ic_density\n",
    "                     for i in range(RESERVOIRS)]\n",
    "                    for j in range(RESERVOIRS)]\n",
    "\n",
    "    IIRENS_model = IIRESN(units=100,\n",
    "                          sub_reservoirs=RESERVOIRS,\n",
    "                          connectivity=connectivity,\n",
    "                          normalization=normalization,\n",
    "                          input_scaling=tuner.Float('input scaling', min_value=MINVAL, max_value=MAXVAL),\n",
    "                          bias_scaling=tuner.Float('bias_scaling', min_value=MINVAL, max_value=MAXVAL),\n",
    "                          leaky=tuner.Float('leaky', min_value=0.01, max_value=1.0),\n",
    "                          output_units=OUTPUT_UNITS,\n",
    "                          readout_activation=READOUT_ACTIVATION,\n",
    "                          )\n",
    "\n",
    "    alpha = tuner.Float('learning rate', min_value=1e-5, max_value=1e-1, sampling='log')\n",
    "    IIRENS_model.compile(\n",
    "        optimizer=keras.optimizers.Adam(alpha),  # keras.optimizers.RMSprop(alpha),\n",
    "        loss=LOSS_FUNCTION,\n",
    "        metrics=['accuracy'],\n",
    "    )\n",
    "    return IIRENS_model"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "In seguito vengono definiti due varianti ai modelli definiti precedentemente\n",
    "- IRESNvsr sub-reseroir di dimensione variabile, definita tramite il parametro di funzione 'vsr'.\n",
    "- IIRESNvsr sub-reseroir di dimensione variabile, definita tramite il parametro di funzione 'vsr'."
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%% md\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "outputs": [],
   "source": [
    "# Generate and Normalize the partition vector. sum(partitions) == 1.\n",
    "def generate_partitions(tuner):\n",
    "    partitions = [tuner.Float('partition ' + str(i), min_value=0.1, max_value=1.0) for i in range(RESERVOIRS)]\n",
    "    total = sum(partitions)\n",
    "    partitions = list(map(lambda _x: 0 if total == 0 else _x / total, partitions))\n",
    "    return partitions\n",
    "\n",
    "\n",
    "def build_IRESNvsr(tuner):\n",
    "    sr = [tuner.Float('spectral radius ' + str(i), min_value=MINVAL, max_value=MAXVAL) for i in range(RESERVOIRS)]\n",
    "    connectivity = [tuner.Float('connectivity ' + str(i), min_value=0., max_value=1.) for i in range(RESERVOIRS)]\n",
    "\n",
    "    IRESNvsr_model = IRESN(units=100,\n",
    "                           sub_reservoirs=RESERVOIRS,\n",
    "                           output_units=OUTPUT_UNITS,\n",
    "                           connectivity=connectivity,\n",
    "                           normalization=sr,\n",
    "                           vsr=generate_partitions(tuner),\n",
    "                           input_scaling=tuner.Float('input scaling', min_value=MINVAL, max_value=MAXVAL),\n",
    "                           bias_scaling=tuner.Float('bias_scaling', min_value=MINVAL, max_value=MAXVAL),\n",
    "                           leaky=tuner.Float('leaky', min_value=0.01, max_value=1.0),\n",
    "                           readout_activation=READOUT_ACTIVATION,\n",
    "                           )\n",
    "\n",
    "    alpha = tuner.Float('learning rate', min_value=1e-5, max_value=1e-1, sampling='log')\n",
    "    IRESNvsr_model.compile(\n",
    "        optimizer=keras.optimizers.Adam(alpha),  # keras.optimizers.RMSprop(alpha),\n",
    "        loss=LOSS_FUNCTION,\n",
    "        metrics=['accuracy'],\n",
    "    )\n",
    "    return IRESNvsr_model\n",
    "\n",
    "\n",
    "def build_IIRESNvsr(tuner):\n",
    "    ic_norm = tuner.Float('norm 2', min_value=MINVAL, max_value=MAXVAL)\n",
    "    normalization = [[tuner.Float('spectral radius ' + str(i), min_value=MINVAL, max_value=MAXVAL) if i == j else\n",
    "                      ic_norm\n",
    "                      for i in range(RESERVOIRS)]\n",
    "                     for j in range(RESERVOIRS)]\n",
    "\n",
    "    ic_density = tuner.Float('connectivity X->Y', min_value=0., max_value=1.)\n",
    "    connectivity = [[tuner.Float('connectivity ' + str(i), min_value=0., max_value=1.) if i == j else\n",
    "                     ic_density\n",
    "                     for i in range(RESERVOIRS)]\n",
    "                    for j in range(RESERVOIRS)]\n",
    "\n",
    "    IIRESNvsr_model = IIRESN(units=100,\n",
    "                             sub_reservoirs=RESERVOIRS,\n",
    "                             connectivity=connectivity,\n",
    "                             normalization=normalization,\n",
    "                             vsr=generate_partitions(tuner),\n",
    "                             input_scaling=tuner.Float('input scaling', min_value=MINVAL, max_value=MAXVAL),\n",
    "                             bias_scaling=tuner.Float('bias_scaling', min_value=MINVAL, max_value=MAXVAL),\n",
    "                             leaky=tuner.Float('leaky', min_value=0.01, max_value=1.0),\n",
    "                             output_units=OUTPUT_UNITS,\n",
    "                             readout_activation=READOUT_ACTIVATION,\n",
    "                             )\n",
    "\n",
    "    alpha = tuner.Float('learning rate', min_value=1e-5, max_value=1e-1, sampling='log')\n",
    "    IIRESNvsr_model.compile(\n",
    "        optimizer=keras.optimizers.Adam(alpha),  # keras.optimizers.RMSprop(alpha),\n",
    "        loss=LOSS_FUNCTION,\n",
    "        metrics=['accuracy'],\n",
    "    )\n",
    "    return IIRESNvsr_model"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "In questi modelli viene tolto il parametro vsr e i valori di normalizzazione dei sub-reservoir ma aggiunto il global spectral radius."
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%% md\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "outputs": [],
   "source": [
    "def build_IRESNgsr(tuner):\n",
    "    connectivity = [tuner.Float('connectivity ' + str(i), min_value=0., max_value=1.) for i in range(RESERVOIRS)]\n",
    "\n",
    "    IRESNvsr_model = IRESN(units=100,\n",
    "                           sub_reservoirs=RESERVOIRS,\n",
    "                           output_units=OUTPUT_UNITS,\n",
    "                           connectivity=connectivity,\n",
    "                           gsr=tuner.Float('global spectral radius', min_value=MINVAL, max_value=MAXVAL),\n",
    "                           input_scaling=tuner.Float('input scaling', min_value=MINVAL, max_value=MAXVAL),\n",
    "                           bias_scaling=tuner.Float('bias_scaling', min_value=MINVAL, max_value=MAXVAL),\n",
    "                           leaky=tuner.Float('leaky', min_value=0.01, max_value=1.0),\n",
    "                           readout_activation=READOUT_ACTIVATION,\n",
    "                           )\n",
    "\n",
    "    alpha = tuner.Float('learning rate', min_value=1e-5, max_value=1e-1, sampling='log')\n",
    "    IRESNvsr_model.compile(\n",
    "        optimizer=keras.optimizers.Adam(alpha),  # keras.optimizers.RMSprop(alpha),\n",
    "        loss=LOSS_FUNCTION,\n",
    "        metrics=['accuracy'],\n",
    "    )\n",
    "    return IRESNvsr_model\n",
    "\n",
    "\n",
    "def build_IIRESNgsr(tuner):\n",
    "    ic_density = tuner.Float('connectivity X->Y', min_value=0., max_value=1.)\n",
    "    connectivity = [[tuner.Float('connectivity ' + str(i), min_value=0., max_value=1.) if i == j else\n",
    "                     ic_density\n",
    "                     for i in range(RESERVOIRS)]\n",
    "                    for j in range(RESERVOIRS)]\n",
    "\n",
    "    IIRESNvsr_model = IIRESN(units=100,\n",
    "                             sub_reservoirs=RESERVOIRS,\n",
    "                             connectivity=connectivity,\n",
    "                             gsr=tuner.Float('global spectral radius', min_value=MINVAL, max_value=MAXVAL),\n",
    "                             input_scaling=tuner.Float('input scaling', min_value=MINVAL, max_value=MAXVAL),\n",
    "                             bias_scaling=tuner.Float('bias_scaling', min_value=MINVAL, max_value=MAXVAL),\n",
    "                             leaky=tuner.Float('leaky', min_value=0.01, max_value=1.0),\n",
    "                             output_units=OUTPUT_UNITS,\n",
    "                             readout_activation=READOUT_ACTIVATION,\n",
    "                             )\n",
    "\n",
    "    alpha = tuner.Float('learning rate', min_value=1e-5, max_value=1e-1, sampling='log')\n",
    "    IIRESNvsr_model.compile(\n",
    "        optimizer=keras.optimizers.Adam(alpha),  # keras.optimizers.RMSprop(alpha),\n",
    "        loss=LOSS_FUNCTION,\n",
    "        metrics=['accuracy'],\n",
    "    )\n",
    "    return IIRESNvsr_model\n"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "Modelli definiti come nella cella precedente ma viene reintrodotta la normalizzazione dei sub-reservoir e rimane il global spectral radius."
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%% md\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "outputs": [],
   "source": [
    "def build_IRESN_norm_gsr(tuner):\n",
    "    connectivity = [tuner.Float('connectivity ' + str(i), min_value=0.01, max_value=1.) for i in range(RESERVOIRS)]\n",
    "    sr = [tuner.Float('spectral radius ' + str(i), min_value=MINVAL, max_value=MAXVAL) for i in range(RESERVOIRS)]\n",
    "\n",
    "    IRESN_model = IRESN(units=100,\n",
    "                        sub_reservoirs=RESERVOIRS,\n",
    "                        connectivity=connectivity,\n",
    "                        normalization=sr,\n",
    "                        gsr=tuner.Float('global spectral radius', min_value=MINVAL, max_value=MAXVAL),\n",
    "                        input_scaling=tuner.Float('input scaling', min_value=MINVAL, max_value=MAXVAL),\n",
    "                        bias_scaling=tuner.Float('bias_scaling', min_value=MINVAL, max_value=MAXVAL),\n",
    "                        leaky=tuner.Float('leaky', min_value=0.01, max_value=1.0),\n",
    "                        output_units=OUTPUT_UNITS,\n",
    "                        readout_activation=READOUT_ACTIVATION,\n",
    "                        )\n",
    "\n",
    "    alpha = tuner.Float('learning rate', min_value=1e-5, max_value=1e-1, sampling='log')\n",
    "    IRESN_model.compile(\n",
    "        optimizer=keras.optimizers.Adam(alpha),  # keras.optimizers.RMSprop(alpha),\n",
    "        loss=LOSS_FUNCTION,\n",
    "        metrics=['accuracy'],\n",
    "    )\n",
    "    return IRESN_model\n",
    "\n",
    "\n",
    "def build_IIRESN_norm_gsr(tuner):\n",
    "    norm = tuner.Float('norm', min_value=MINVAL, max_value=MAXVAL)\n",
    "    normalization = [[tuner.Float('spectral radius ' + str(i), min_value=MINVAL, max_value=MAXVAL) if i == j else norm\n",
    "                      # 0 sulla diagonale limit su tutte le posizioni off-diagonali\n",
    "                      for i in range(RESERVOIRS)]\n",
    "                     for j in range(RESERVOIRS)]\n",
    "\n",
    "    ic_density = tuner.Float('connectivity X->Y', min_value=0., max_value=1.)\n",
    "    connectivity = [[tuner.Float('connectivity ' + str(i), min_value=0., max_value=1.) if i == j else\n",
    "                     ic_density\n",
    "                     for i in range(RESERVOIRS)]\n",
    "                    for j in range(RESERVOIRS)]\n",
    "\n",
    "    IIRENS_model = IIRESN(units=100,\n",
    "                          sub_reservoirs=RESERVOIRS,\n",
    "                          connectivity=connectivity,\n",
    "                          normalization=normalization,\n",
    "                          gsr=tuner.Float('global spectral radius', min_value=MINVAL, max_value=MAXVAL),\n",
    "                          input_scaling=tuner.Float('input scaling', min_value=MINVAL, max_value=MAXVAL),\n",
    "                          bias_scaling=tuner.Float('bias_scaling', min_value=MINVAL, max_value=MAXVAL),\n",
    "                          leaky=tuner.Float('leaky', min_value=0.01, max_value=1.0),\n",
    "                          output_units=OUTPUT_UNITS,\n",
    "                          readout_activation=READOUT_ACTIVATION,\n",
    "                          )\n",
    "\n",
    "    alpha = tuner.Float('learning rate', min_value=1e-5, max_value=1e-1, sampling='log')\n",
    "    IIRENS_model.compile(\n",
    "        optimizer=keras.optimizers.Adam(alpha),  # keras.optimizers.RMSprop(alpha),\n",
    "        loss=LOSS_FUNCTION,\n",
    "        metrics=['accuracy'],\n",
    "    )\n",
    "    return IIRENS_model"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/dati/luca/Uni-Luca/Tesi/progetto/venv/lib/python3.9/site-packages/sktime/utils/data_io.py:63: FutureWarning: This function has moved to datasets/_data_io, this version will be removed in V0.10\n",
      "  warn(\n",
      "/dati/luca/Uni-Luca/Tesi/progetto/venv/lib/python3.9/site-packages/sktime/utils/data_io.py:63: FutureWarning: This function has moved to datasets/_data_io, this version will be removed in V0.10\n",
      "  warn(\n"
     ]
    }
   ],
   "source": [
    "train_path = os.path.join(DATASET_DIR, DATASET_NAME, DATASET_NAME + '_TRAIN.ts')\n",
    "test_path = os.path.join(DATASET_DIR, DATASET_NAME, DATASET_NAME + '_TEST.ts')\n",
    "\n",
    "x_train_all, y_train_all = load_sktime_dataset(train_path)\n",
    "x_test, y_test = load_sktime_dataset(test_path)\n",
    "\n",
    "x_train, x_val, y_train, y_val = train_test_split(x_train_all, y_train_all,\n",
    "                                                  test_size=0.33, random_state=42, stratify=y_train_all)\n",
    "\n",
    "train_set = (x_train, y_train)\n",
    "val_set = (x_val, y_val)\n",
    "test_set = (x_test, y_test)"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Search: Running Trial #1\n",
      "\n",
      "Hyperparameter    |Value             |Best Value So Far \n",
      "connectivity      |0.64303           |?                 \n",
      "spectral radius   |0.067442          |?                 \n",
      "input scaling     |0.61872           |?                 \n",
      "bias scaling      |0.41509           |?                 \n",
      "leaky             |0.88939           |?                 \n",
      "learning rate     |0.0002555         |?                 \n",
      "\n",
      "Inputs (470, 3)\n",
      "Inputs (470, 3)\n"
     ]
    },
    {
     "ename": "AttributeError",
     "evalue": "Exception encountered when calling layer \"reservoir\" (type Reservoir).\n\nin user code:\n\n    File \"/dati/luca/Uni-Luca/Tesi/IRESNs-tensorflow/IRESNs_tensorflow/layer.py\", line 180, in call  *\n        print(\"states\", states[0].numpy())\n\n    AttributeError: 'Tensor' object has no attribute 'numpy'\n\n\nCall arguments received:\n  • inputs=tf.Tensor(shape=(470, 3), dtype=float32)\n  • states=('tf.Tensor(shape=(470, 100), dtype=float32)',)",
     "output_type": "error",
     "traceback": [
      "\u001B[0;31m---------------------------------------------------------------------------\u001B[0m",
      "\u001B[0;31mAttributeError\u001B[0m                            Traceback (most recent call last)",
      "Input \u001B[0;32mIn [24]\u001B[0m, in \u001B[0;36m<module>\u001B[0;34m\u001B[0m\n\u001B[1;32m      5\u001B[0m     os\u001B[38;5;241m.\u001B[39mmakedirs(working_dir)\n\u001B[1;32m      7\u001B[0m tuner \u001B[38;5;241m=\u001B[39m RandomSearch(\n\u001B[1;32m      8\u001B[0m     build_ESN,\n\u001B[1;32m      9\u001B[0m     \u001B[38;5;66;03m# build_ESN\u001B[39;00m\n\u001B[0;32m   (...)\u001B[0m\n\u001B[1;32m     18\u001B[0m     executions_per_trial\u001B[38;5;241m=\u001B[39mTRIALS,\n\u001B[1;32m     19\u001B[0m )\n\u001B[0;32m---> 21\u001B[0m \u001B[43mtuner\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43msearch\u001B[49m\u001B[43m(\u001B[49m\u001B[43mx_train\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43my_train\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43mepochs\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43mMAX_EPOCHS\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43mvalidation_data\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43m(\u001B[49m\u001B[43mx_val\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43my_val\u001B[49m\u001B[43m)\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m     22\u001B[0m \u001B[43m             \u001B[49m\u001B[43mcallbacks\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43m[\u001B[49m\n\u001B[1;32m     23\u001B[0m \u001B[43m                 \u001B[49m\u001B[43mkeras\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mcallbacks\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mEarlyStopping\u001B[49m\u001B[43m(\u001B[49m\u001B[43mmonitor\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[38;5;124;43m'\u001B[39;49m\u001B[38;5;124;43mval_loss\u001B[39;49m\u001B[38;5;124;43m'\u001B[39;49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43mpatience\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43mPATIENCE\u001B[49m\u001B[43m)\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m     24\u001B[0m \u001B[43m             \u001B[49m\u001B[43m]\u001B[49m\u001B[43m)\u001B[49m\n\u001B[1;32m     26\u001B[0m best_model_hp \u001B[38;5;241m=\u001B[39m tuner\u001B[38;5;241m.\u001B[39mget_best_hyperparameters()[\u001B[38;5;241m0\u001B[39m]\n\u001B[1;32m     27\u001B[0m tf\u001B[38;5;241m.\u001B[39mrandom\u001B[38;5;241m.\u001B[39mset_seed(\u001B[38;5;241m42\u001B[39m)\n",
      "File \u001B[0;32m/dati/luca/Uni-Luca/Tesi/progetto/venv/lib/python3.9/site-packages/keras_tuner/engine/base_tuner.py:179\u001B[0m, in \u001B[0;36mBaseTuner.search\u001B[0;34m(self, *fit_args, **fit_kwargs)\u001B[0m\n\u001B[1;32m    176\u001B[0m     \u001B[38;5;28;01mcontinue\u001B[39;00m\n\u001B[1;32m    178\u001B[0m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39mon_trial_begin(trial)\n\u001B[0;32m--> 179\u001B[0m results \u001B[38;5;241m=\u001B[39m \u001B[38;5;28;43mself\u001B[39;49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mrun_trial\u001B[49m\u001B[43m(\u001B[49m\u001B[43mtrial\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[38;5;241;43m*\u001B[39;49m\u001B[43mfit_args\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[38;5;241;43m*\u001B[39;49m\u001B[38;5;241;43m*\u001B[39;49m\u001B[43mfit_kwargs\u001B[49m\u001B[43m)\u001B[49m\n\u001B[1;32m    180\u001B[0m \u001B[38;5;66;03m# `results` is None indicates user updated oracle in `run_trial()`.\u001B[39;00m\n\u001B[1;32m    181\u001B[0m \u001B[38;5;28;01mif\u001B[39;00m results \u001B[38;5;129;01mis\u001B[39;00m \u001B[38;5;28;01mNone\u001B[39;00m:\n",
      "File \u001B[0;32m/dati/luca/Uni-Luca/Tesi/progetto/venv/lib/python3.9/site-packages/keras_tuner/engine/tuner.py:304\u001B[0m, in \u001B[0;36mTuner.run_trial\u001B[0;34m(self, trial, *args, **kwargs)\u001B[0m\n\u001B[1;32m    302\u001B[0m callbacks\u001B[38;5;241m.\u001B[39mappend(model_checkpoint)\n\u001B[1;32m    303\u001B[0m copied_kwargs[\u001B[38;5;124m\"\u001B[39m\u001B[38;5;124mcallbacks\u001B[39m\u001B[38;5;124m\"\u001B[39m] \u001B[38;5;241m=\u001B[39m callbacks\n\u001B[0;32m--> 304\u001B[0m obj_value \u001B[38;5;241m=\u001B[39m \u001B[38;5;28;43mself\u001B[39;49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43m_build_and_fit_model\u001B[49m\u001B[43m(\u001B[49m\u001B[43mtrial\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[38;5;241;43m*\u001B[39;49m\u001B[43margs\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[38;5;241;43m*\u001B[39;49m\u001B[38;5;241;43m*\u001B[39;49m\u001B[43mcopied_kwargs\u001B[49m\u001B[43m)\u001B[49m\n\u001B[1;32m    306\u001B[0m \u001B[38;5;66;03m# objective left unspecified,\u001B[39;00m\n\u001B[1;32m    307\u001B[0m \u001B[38;5;66;03m# and objective value is not a single float.\u001B[39;00m\n\u001B[1;32m    308\u001B[0m \u001B[38;5;28;01mif\u001B[39;00m (\n\u001B[1;32m    309\u001B[0m     \u001B[38;5;129;01mnot\u001B[39;00m \u001B[38;5;28misinstance\u001B[39m(obj_value, (\u001B[38;5;28mint\u001B[39m, \u001B[38;5;28mfloat\u001B[39m))\n\u001B[1;32m    310\u001B[0m     \u001B[38;5;129;01mand\u001B[39;00m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39moracle\u001B[38;5;241m.\u001B[39mobjective\u001B[38;5;241m.\u001B[39mname \u001B[38;5;241m==\u001B[39m \u001B[38;5;124m\"\u001B[39m\u001B[38;5;124mdefault_objective\u001B[39m\u001B[38;5;124m\"\u001B[39m\n\u001B[1;32m    311\u001B[0m ):\n",
      "File \u001B[0;32m/dati/luca/Uni-Luca/Tesi/progetto/venv/lib/python3.9/site-packages/keras_tuner/engine/tuner.py:234\u001B[0m, in \u001B[0;36mTuner._build_and_fit_model\u001B[0;34m(self, trial, *args, **kwargs)\u001B[0m\n\u001B[1;32m    232\u001B[0m hp \u001B[38;5;241m=\u001B[39m trial\u001B[38;5;241m.\u001B[39mhyperparameters\n\u001B[1;32m    233\u001B[0m model \u001B[38;5;241m=\u001B[39m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39m_try_build(hp)\n\u001B[0;32m--> 234\u001B[0m \u001B[38;5;28;01mreturn\u001B[39;00m \u001B[38;5;28;43mself\u001B[39;49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mhypermodel\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mfit\u001B[49m\u001B[43m(\u001B[49m\u001B[43mhp\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43mmodel\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[38;5;241;43m*\u001B[39;49m\u001B[43margs\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[38;5;241;43m*\u001B[39;49m\u001B[38;5;241;43m*\u001B[39;49m\u001B[43mkwargs\u001B[49m\u001B[43m)\u001B[49m\n",
      "File \u001B[0;32m/dati/luca/Uni-Luca/Tesi/progetto/venv/lib/python3.9/site-packages/keras_tuner/engine/hypermodel.py:137\u001B[0m, in \u001B[0;36mHyperModel.fit\u001B[0;34m(self, hp, model, *args, **kwargs)\u001B[0m\n\u001B[1;32m    113\u001B[0m \u001B[38;5;28;01mdef\u001B[39;00m \u001B[38;5;21mfit\u001B[39m(\u001B[38;5;28mself\u001B[39m, hp, model, \u001B[38;5;241m*\u001B[39margs, \u001B[38;5;241m*\u001B[39m\u001B[38;5;241m*\u001B[39mkwargs):\n\u001B[1;32m    114\u001B[0m     \u001B[38;5;124;03m\"\"\"Train the model.\u001B[39;00m\n\u001B[1;32m    115\u001B[0m \n\u001B[1;32m    116\u001B[0m \u001B[38;5;124;03m    Args:\u001B[39;00m\n\u001B[0;32m   (...)\u001B[0m\n\u001B[1;32m    135\u001B[0m \u001B[38;5;124;03m        If return a float, it should be the `objective` value.\u001B[39;00m\n\u001B[1;32m    136\u001B[0m \u001B[38;5;124;03m    \"\"\"\u001B[39;00m\n\u001B[0;32m--> 137\u001B[0m     \u001B[38;5;28;01mreturn\u001B[39;00m \u001B[43mmodel\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mfit\u001B[49m\u001B[43m(\u001B[49m\u001B[38;5;241;43m*\u001B[39;49m\u001B[43margs\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[38;5;241;43m*\u001B[39;49m\u001B[38;5;241;43m*\u001B[39;49m\u001B[43mkwargs\u001B[49m\u001B[43m)\u001B[49m\n",
      "File \u001B[0;32m/dati/luca/Uni-Luca/Tesi/IRESNs-tensorflow/IRESNs_tensorflow/models.py:35\u001B[0m, in \u001B[0;36mESNInterface.fit\u001B[0;34m(self, x, y, *args, **kwargs)\u001B[0m\n\u001B[1;32m     32\u001B[0m \u001B[38;5;28;01mdef\u001B[39;00m \u001B[38;5;21mfit\u001B[39m(\u001B[38;5;28mself\u001B[39m, x, y, \u001B[38;5;241m*\u001B[39margs, \u001B[38;5;241m*\u001B[39m\u001B[38;5;241m*\u001B[39mkwargs):\n\u001B[1;32m     33\u001B[0m     \u001B[38;5;66;03m# does the same for the validation set\u001B[39;00m\n\u001B[1;32m     34\u001B[0m     x_val, y_val \u001B[38;5;241m=\u001B[39m kwargs[\u001B[38;5;124m'\u001B[39m\u001B[38;5;124mvalidation_data\u001B[39m\u001B[38;5;124m'\u001B[39m]\n\u001B[0;32m---> 35\u001B[0m     x_val_out \u001B[38;5;241m=\u001B[39m \u001B[38;5;28;43mself\u001B[39;49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mreservoir\u001B[49m\u001B[43m(\u001B[49m\u001B[43mx_val\u001B[49m\u001B[43m)\u001B[49m\n\u001B[1;32m     36\u001B[0m     kwargs[\u001B[38;5;124m'\u001B[39m\u001B[38;5;124mvalidation_data\u001B[39m\u001B[38;5;124m'\u001B[39m] \u001B[38;5;241m=\u001B[39m (x_val_out, y_val)\n\u001B[1;32m     38\u001B[0m     \u001B[38;5;66;03m# applies the reservoirs to all the input sequences in the training set\u001B[39;00m\n",
      "File \u001B[0;32m/dati/luca/Uni-Luca/Tesi/progetto/venv/lib/python3.9/site-packages/keras/utils/traceback_utils.py:67\u001B[0m, in \u001B[0;36mfilter_traceback.<locals>.error_handler\u001B[0;34m(*args, **kwargs)\u001B[0m\n\u001B[1;32m     65\u001B[0m \u001B[38;5;28;01mexcept\u001B[39;00m \u001B[38;5;167;01mException\u001B[39;00m \u001B[38;5;28;01mas\u001B[39;00m e:  \u001B[38;5;66;03m# pylint: disable=broad-except\u001B[39;00m\n\u001B[1;32m     66\u001B[0m   filtered_tb \u001B[38;5;241m=\u001B[39m _process_traceback_frames(e\u001B[38;5;241m.\u001B[39m__traceback__)\n\u001B[0;32m---> 67\u001B[0m   \u001B[38;5;28;01mraise\u001B[39;00m e\u001B[38;5;241m.\u001B[39mwith_traceback(filtered_tb) \u001B[38;5;28;01mfrom\u001B[39;00m \u001B[38;5;28mNone\u001B[39m\n\u001B[1;32m     68\u001B[0m \u001B[38;5;28;01mfinally\u001B[39;00m:\n\u001B[1;32m     69\u001B[0m   \u001B[38;5;28;01mdel\u001B[39;00m filtered_tb\n",
      "File \u001B[0;32m/dati/luca/Uni-Luca/Tesi/IRESNs-tensorflow/IRESNs_tensorflow/layer.py:50\u001B[0m, in \u001B[0;36mESN.call\u001B[0;34m(self, inputs, mask, training, initial_state)\u001B[0m\n\u001B[1;32m     49\u001B[0m \u001B[38;5;28;01mdef\u001B[39;00m \u001B[38;5;21mcall\u001B[39m(\u001B[38;5;28mself\u001B[39m, inputs, mask\u001B[38;5;241m=\u001B[39m\u001B[38;5;28;01mNone\u001B[39;00m, training\u001B[38;5;241m=\u001B[39m\u001B[38;5;28;01mNone\u001B[39;00m, initial_state\u001B[38;5;241m=\u001B[39m\u001B[38;5;28;01mNone\u001B[39;00m):\n\u001B[0;32m---> 50\u001B[0m     \u001B[38;5;28;01mreturn\u001B[39;00m \u001B[38;5;28;43msuper\u001B[39;49m\u001B[43m(\u001B[49m\u001B[43m)\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mcall\u001B[49m\u001B[43m(\u001B[49m\n\u001B[1;32m     51\u001B[0m \u001B[43m        \u001B[49m\u001B[43minputs\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m     52\u001B[0m \u001B[43m        \u001B[49m\u001B[43mmask\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43mmask\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m     53\u001B[0m \u001B[43m        \u001B[49m\u001B[43mtraining\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43mtraining\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m     54\u001B[0m \u001B[43m        \u001B[49m\u001B[43minitial_state\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43minitial_state\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m     55\u001B[0m \u001B[43m        \u001B[49m\u001B[43mconstants\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[38;5;28;43;01mNone\u001B[39;49;00m\n\u001B[1;32m     56\u001B[0m \u001B[43m    \u001B[49m\u001B[43m)\u001B[49m\n",
      "\u001B[0;31mAttributeError\u001B[0m: Exception encountered when calling layer \"reservoir\" (type Reservoir).\n\nin user code:\n\n    File \"/dati/luca/Uni-Luca/Tesi/IRESNs-tensorflow/IRESNs_tensorflow/layer.py\", line 180, in call  *\n        print(\"states\", states[0].numpy())\n\n    AttributeError: 'Tensor' object has no attribute 'numpy'\n\n\nCall arguments received:\n  • inputs=tf.Tensor(shape=(470, 3), dtype=float32)\n  • states=('tf.Tensor(shape=(470, 100), dtype=float32)',)"
     ]
    }
   ],
   "source": [
    "from keras_tuner import RandomSearch\n",
    "\n",
    "working_dir = os.path.join(\"models\", DATASET_NAME)\n",
    "if not os.path.exists(working_dir):\n",
    "    os.makedirs(working_dir)\n",
    "\n",
    "tuner = RandomSearch(\n",
    "    build_ESN,\n",
    "    # build_ESN\n",
    "    # build_IRESN, build_IRESNvsr, build_IRESNgsr, build_IRESN_norm_gsr,\n",
    "    # build_IIRESN, build_IIRESNvsr, build_IIRESNvsr, build_IIRESN_norm_gsr\n",
    "    objective='val_accuracy',\n",
    "    max_trials=MAX_TRIALS,\n",
    "    seed=42,\n",
    "    directory=working_dir,\n",
    "    project_name='build_IIRESN',  # change this based on the build function used\n",
    "    overwrite=OVERWRITE,\n",
    "    executions_per_trial=TRIALS,\n",
    ")\n",
    "\n",
    "tuner.search(x_train, y_train, epochs=MAX_EPOCHS, validation_data=(x_val, y_val),\n",
    "             callbacks=[\n",
    "                 keras.callbacks.EarlyStopping(monitor='val_loss', patience=PATIENCE),\n",
    "             ])\n",
    "\n",
    "best_model_hp = tuner.get_best_hyperparameters()[0]\n",
    "tf.random.set_seed(42)\n",
    "\n",
    "for i in range(BENCHMARKS_TRIALS):\n",
    "    test_model = tuner.hypermodel.build(best_model_hp)\n",
    "    history = test_model.fit(x_train, y_train, validation_data=(x_val, y_val), epochs=MAX_EPOCHS,\n",
    "                             callbacks=[keras.callbacks.EarlyStopping(monitor='val_loss', patience=PATIENCE,\n",
    "                                                                      restore_best_weights=True)])\n",
    "    test_loss, accuracy = test_model.evaluate(x_test, y_test)\n",
    "\n",
    "    print(\"     Train accuracy:\", history.history['accuracy'][-1])\n",
    "    print(\"Validation accuracy:\", history.history['val_accuracy'][-1])\n",
    "    print(\"      Test accuracy:\", accuracy)"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "# Build some custom Models"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%% md\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "outputs": [],
   "source": [
    "\n",
    "\"\"\"\n",
    "Experimental model with kernel equals to IRESN and recurrent kernel equals to ESN.\n",
    "\"\"\"\n",
    "\n",
    "from IRESNs_tensorflow.models import ESNInterface\n",
    "from IRESNs_tensorflow import layer\n",
    "from IRESNs_tensorflow import initializers\n",
    "\n",
    "\n",
    "class CustomESN(ESNInterface):\n",
    "    def __init__(self,\n",
    "                 units: int,\n",
    "                 sub_reservoirs: int,\n",
    "                 output_units: int,\n",
    "                 reservoir_activation=tf.keras.activations.tanh,\n",
    "                 readout_activation=tf.nn.tanh,\n",
    "                 spectral_radius: float = 0.9,\n",
    "                 connectivity: float = 1.,\n",
    "                 input_scaling: float = 1.,\n",
    "                 bias_scaling=None,\n",
    "                 leaky=0.1,\n",
    "                 **kwargs\n",
    "                 ):\n",
    "        super().__init__(**kwargs)\n",
    "\n",
    "        kernel_init = initializers.SplitKernel(sub_reservoirs, input_scaling)\n",
    "\n",
    "        if connectivity == 1.0:\n",
    "            recurrent_kernel_init = initializers.FullConnected(spectral_radius)\n",
    "        else:\n",
    "            recurrent_kernel_init = initializers.RecurrentKernel(connectivity, spectral_radius)\n",
    "\n",
    "        self.use_bias = bias_scaling is not None\n",
    "        if self.use_bias:\n",
    "            bias_init = tf.keras.initializers.RandomUniform(minval=-bias_scaling, maxval=bias_scaling)\n",
    "        else:\n",
    "            bias_init = None\n",
    "\n",
    "        self.reservoir = keras.Sequential([\n",
    "            keras.layers.Masking(),\n",
    "            layer.ESN(units, leaky,\n",
    "                      activation=reservoir_activation,\n",
    "                      use_bias=self.use_bias,\n",
    "                      kernel_initializer=kernel_init,\n",
    "                      recurrent_initializer=recurrent_kernel_init,\n",
    "                      bias_initializer=bias_init\n",
    "                      ),\n",
    "        ])\n",
    "        self.readout = keras.Sequential([\n",
    "            keras.layers.Dense(output_units, activation=readout_activation, name=\"readout\")\n",
    "        ])\n",
    "\n",
    "\n",
    "def build_CustomESN(tuner):\n",
    "    custom_model = CustomESN(units=100,\n",
    "                             sub_reservoirs=RESERVOIRS,\n",
    "                             connectivity=tuner.Float('connectivity', min_value=0.01, max_value=1.),\n",
    "                             spectral_radius=tuner.Float('spectral radius', min_value=MINVAL, max_value=MAXVAL),\n",
    "                             input_scaling=tuner.Float('input scaling', min_value=MINVAL, max_value=MAXVAL),\n",
    "                             bias_scaling=tuner.Float('bias scaling', min_value=MINVAL, max_value=MAXVAL),\n",
    "                             leaky=tuner.Float('leaky', min_value=0.01, max_value=1.0),\n",
    "                             output_units=OUTPUT_UNITS,\n",
    "                             readout_activation=READOUT_ACTIVATION\n",
    "                             )\n",
    "\n",
    "    alpha = tuner.Float('learning rate', min_value=1e-5, max_value=1e-1, sampling='log')\n",
    "    custom_model.compile(\n",
    "        optimizer=keras.optimizers.Adam(alpha),\n",
    "        loss=LOSS_FUNCTION,\n",
    "        metrics=['accuracy'],\n",
    "    )\n",
    "    return custom_model"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Trial 5 Complete [00h 00m 00s]\n",
      "val_accuracy: 0.06382978707551956\n",
      "\n",
      "Best val_accuracy So Far: 0.2638297975063324\n",
      "Total elapsed time: 00h 00m 04s\n",
      "INFO:tensorflow:Oracle triggered exit\n"
     ]
    }
   ],
   "source": [
    "tuner = RandomSearch(\n",
    "    build_CustomESN,  # build_ESN build_IRESN, build_IIRESN, build_IIRESNvsr\n",
    "    objective='val_accuracy',\n",
    "    max_trials=MAX_TRIALS,\n",
    "    seed=42,\n",
    "    directory=working_dir,\n",
    "    project_name='CustomESN',  # change this for every model\n",
    "    overwrite=OVERWRITE,\n",
    "    executions_per_trial=TRIALS,\n",
    ")\n",
    "\n",
    "tuner.search(x_train, y_train, epochs=MAX_EPOCHS, validation_data=(x_val, y_val),\n",
    "             callbacks=[\n",
    "                 keras.callbacks.EarlyStopping(monitor='val_loss', patience=PATIENCE),\n",
    "             ])"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  }
 ],
 "metadata": {
  "kernelspec": {
   "name": "venv",
   "language": "python",
   "display_name": "venv"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}